From a7f5568ee73beaa1b4b036efe07a70faaaf71066 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Sat, 1 Aug 2026 10:34:11 +0200 Subject: [PATCH 1/4] feat(auth): require staff KYC clearance on elevated endpoints (#4395) * feat(auth): require staff KYC clearance on elevated endpoints Access to endpoints gated on Admin, Debug, Compliance, Support or RealUnit now additionally requires an identified natural person behind the calling account: kycLevel >= 50 and a non-empty verifiedName. The check lives in RoleGuard, so it covers all 235 elevated gates at once. It is a property of the endpoint, not of the caller: it applies only when every entry role of a gate is elevated, so an admin using ordinary customer functionality is unaffected. StaffKycClearanceService derives the cleared account ids from the DB every minute, ProcessService primes an in-memory Set from the resulting setting, and RoleGuard reads it synchronously. This mirrors the existing JWT denylists: the requirement takes effect on already-issued tokens within one refresh interval, with no DB lookup per request. The allowlist is deliberately fail-closed - an empty or not-yet-primed Set denies every elevated endpoint, so a DB or cron outage cannot silently re-open admin access. Three privilege checks that live in business logic rather than a gate are covered too, via the new hasStaffAccess helper. All three sit on routes that are not role-gated (OptionalJwtAuthGuard), so the requirement would otherwise be bypassable: protected KYC file downloads, ownership-independent access to any customer's transaction history, and posting official support replies. Privilege checks behind already-gated endpoints (the isAdmin distinctions in the support note services) are left unchanged - the gate has run by then. The three files deciding elevated access are pinned at 100% coverage on all four metrics via a dedicated CI step, following the existing Frick coverage gate. * fix(auth): correct import order and a stale service reference Restores the alphabetical import order in user.module.ts and fixes a comment in setting.service.ts that referenced a class name that does not exist. * refactor(auth): filter the verified name in SQL and fix import order Moves the verifiedName condition out of JS and into the query, per the "filter in SQL, not JS" rule: TRIM(...) <> '' covers empty and whitespace-only names and drops NULL on its own, since the comparison yields NULL. The raw fragment interpolates the alias TypeORM passes in, which is already quoted - an unquoted camelCase identifier would be folded to lowercase by Postgres and fail at runtime. Verified against a real Postgres instance: the emitted SQL is TRIM("..."."verifiedName") <> '' and only genuinely verified staff rows survive, with NULL, empty and whitespace-only names all filtered out. The unit test pins the rendered predicate, since mocked repositories never execute the fragment. Also restores the alphabetical import order in process.service.ts. * fix(auth): close two gaps in the clearance path Two holes found in review, both in the path that decides elevated access. Blank names: Postgres TRIM() with no character argument strips ASCII space only, while the JS trim() it replaced also strips tabs, newlines and Unicode spaces. A verifiedName of a single tab or a non-breaking space therefore passed the filter and cleared an account carrying no identification. The predicate now uses BTRIM with the explicit character set that trim() strips, spelled out rather than left to [[:space:]], whose meaning depends on the database locale. A mocked repository never executes raw SQL, so a new suite runs the predicate against a real Postgres and pins it to the semantics it replaced: every blank variant is excluded, padded real names are kept, and the result is asserted to equal the JS filter on every input. It follows the existing MIGRATION_TEST_PG pattern, so CI runs it against its throwaway database. Manual override: the generic PUT /setting/:key route accepted staffKycClearance, which contradicted this branch's own comment and let a cleared admin write uncleared accounts into the allowlist, live until the next sync overwrote it. Sync-owned settings are now rejected there; the sync path writes through setObj and is unaffected. * refactor(setting): enforce system-managed keys in the service Moves the system-managed check from the controller into SettingService.set, where it covers every caller rather than only the HTTP route, and keeps the controller thin as the contributing guide requires. The sync path writes through setObj, which goes to the repository directly, so it stays unaffected - now pinned by a test. Widens the Postgres suite so the character set itself is guarded: the blank fixtures are derived from the JS runtime rather than from BlankChars, because fixtures generated from the constant under test would shrink along with it and could never catch a character dropped from it. Verified by removing one character from BlankChars, which turns the suite red. * test(setting): pin the key and value written by the generic setter Pins key and value on the generic setter instead of only asserting that something was written - a swapped argument pair would otherwise pass, and this is the only place the pair is checked since the controller test was folded into the service test. Records why U+200B is absent from BlankChars: trim() does not strip it either, so adding it would make the predicate stricter than the check it replaced. Whether a name made of invisible characters counts as identification is a question about how verifiedName is written, not about this gate. --- .github/workflows/api-pr.yaml | 5 + jest.staff-gate.config.js | 26 +++ package.json | 1 + src/shared/auth/__tests__/role.guard.spec.ts | 153 +++++++++++++++++- .../__tests__/staff-kyc-clearance.spec.ts | 46 ++++++ src/shared/auth/role.guard.ts | 47 +++++- src/shared/auth/staff-kyc-clearance.ts | 26 +++ src/shared/auth/user-role.enum.ts | 10 ++ .../setting/__tests__/setting.service.spec.ts | 61 ++++++- src/shared/models/setting/setting.service.ts | 22 ++- .../__tests__/process.service.spec.ts | 36 +++++ src/shared/services/process.service.ts | 11 ++ .../__tests__/history-access.service.spec.ts | 27 ++++ .../services/history-access.service.ts | 14 +- .../services/__tests__/kyc.service.spec.ts | 27 ++++ .../generic/kyc/services/kyc.service.ts | 8 +- .../__tests__/staff-kyc-clearance.pg.spec.ts | 118 ++++++++++++++ .../staff-kyc-clearance.service.spec.ts | 103 ++++++++++++ .../user/staff-kyc-clearance.service.ts | 74 +++++++++ src/subdomains/generic/user/user.module.ts | 2 + .../support-issue.controller.spec.ts | 21 +++ .../support-issue/support-issue.controller.ts | 6 +- 22 files changed, 825 insertions(+), 19 deletions(-) create mode 100644 jest.staff-gate.config.js create mode 100644 src/shared/auth/__tests__/staff-kyc-clearance.spec.ts create mode 100644 src/shared/auth/staff-kyc-clearance.ts create mode 100644 src/subdomains/generic/user/models/user/__tests__/staff-kyc-clearance.pg.spec.ts create mode 100644 src/subdomains/generic/user/models/user/__tests__/staff-kyc-clearance.service.spec.ts create mode 100644 src/subdomains/generic/user/models/user/staff-kyc-clearance.service.ts diff --git a/.github/workflows/api-pr.yaml b/.github/workflows/api-pr.yaml index 52675a7cba..2e1b12ec70 100644 --- a/.github/workflows/api-pr.yaml +++ b/.github/workflows/api-pr.yaml @@ -83,6 +83,11 @@ jobs: - name: Run coverage run: npm run test:frick:cov + # Same mechanics for the staff KYC gate: the files deciding who reaches an elevated endpoint are + # pinned at 100%, so an uncovered branch in the authorization path fails the PR. + - name: Run staff gate coverage + run: npm run test:staff-gate:cov + # Runs on a self-hosted runner for branches of this repository. The gate executes the whole suite # under full compilation and is CPU-bound; a hosted runner gives a public repo four vCPUs, so Jest # defaults to three workers and the gate alone decided how long a PR run took. The self-hosted diff --git a/jest.staff-gate.config.js b/jest.staff-gate.config.js new file mode 100644 index 0000000000..5c117b1a0b --- /dev/null +++ b/jest.staff-gate.config.js @@ -0,0 +1,26 @@ +// Staff KYC gate coverage. Kept out of package.json's shared Jest config (same reasoning as +// jest.frick.config.js) so the strict per-file 100% threshold cannot red an unrelated `test:cov` run — +// only the dedicated test:staff-gate:cov step, with its own --collectCoverageFrom scope, enforces it. +// +// These three files decide who reaches every elevated endpoint. A partially covered branch here is an +// unreviewed hole in the authorization path, so they are pinned at 100% on all four metrics. +const base = require('./package.json').jest; + +module.exports = { + ...base, + // Coverage instrumentation must match the production build's emit. The main suite runs ts-jest in + // transpile-only mode (isolatedModules), which emits the emitDecoratorMetadata helpers differently + // and produces phantom uncovered branches on dependency-injected constructors. Compile with full + // type info here (tsconfig.coverage.json sets isolatedModules: false) so the 100% gate stays exact. + transform: { '^.+\\.(t|j)s$': ['ts-jest', { tsconfig: 'tsconfig.coverage.json' }] }, + coverageThreshold: { + 'src/shared/auth/role.guard.ts': { branches: 100, functions: 100, lines: 100, statements: 100 }, + 'src/shared/auth/staff-kyc-clearance.ts': { branches: 100, functions: 100, lines: 100, statements: 100 }, + 'src/subdomains/generic/user/models/user/staff-kyc-clearance.service.ts': { + branches: 100, + functions: 100, + lines: 100, + statements: 100, + }, + }, +}; diff --git a/package.json b/package.json index 74a9f61938..8161508d2d 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "test:cov": "jest --coverage", "test:frick:cov": "jest --config jest.frick.config.js integration/bank/services/__tests__/frick.service.spec.ts integration/bank/services/__tests__/iso20022.service.spec.ts config/__tests__/frick.config.spec.ts config/__tests__/bank-frick-config.spec.ts subdomains/supporting/bank-tx/bank-tx/services/__tests__/bank-tx-frick.service.spec.ts subdomains/supporting/bank-tx/bank-tx/services/__tests__/bank-tx-outgoing-match.service.spec.ts subdomains/supporting/fiat-output/__tests__/fiat-output-frick.service.spec.ts subdomains/supporting/bank/virtual-iban/__tests__/virtual-iban-frick-issuance-reconciliation.service.spec.ts subdomains/supporting/bank/virtual-iban/__tests__/virtual-iban.service.spec.ts subdomains/supporting/bank/virtual-iban/providers/__tests__/frick-viban.provider.spec.ts --coverage --runInBand --collectCoverageFrom=integration/bank/dto/frick.dto.ts --collectCoverageFrom=integration/bank/services/frick.service.ts --collectCoverageFrom=integration/bank/services/iso20022.service.ts --collectCoverageFrom=config/frick.config.ts --collectCoverageFrom=subdomains/supporting/bank-tx/bank-tx/services/bank-tx-frick.service.ts --collectCoverageFrom=subdomains/supporting/bank-tx/bank-tx/services/bank-tx-outgoing-match.service.ts --collectCoverageFrom=subdomains/supporting/fiat-output/fiat-output-frick.service.ts --collectCoverageFrom=subdomains/supporting/bank/virtual-iban/virtual-iban-frick-issuance-reconciliation.service.ts --collectCoverageFrom=subdomains/supporting/bank/virtual-iban/virtual-iban.service.ts --collectCoverageFrom=subdomains/supporting/bank/virtual-iban/providers/frick-viban.provider.ts", "test:gate:cov": "jest --config jest.coverage-gate.config.js --coverage --silent", + "test:staff-gate:cov": "jest --config jest.staff-gate.config.js shared/auth/__tests__/role.guard.spec.ts shared/auth/__tests__/staff-kyc-clearance.spec.ts subdomains/generic/user/models/user/__tests__/staff-kyc-clearance.service.spec.ts --coverage --runInBand --collectCoverageFrom=shared/auth/role.guard.ts --collectCoverageFrom=shared/auth/staff-kyc-clearance.ts --collectCoverageFrom=subdomains/generic/user/models/user/staff-kyc-clearance.service.ts", "type-check": "tsc --noEmit", "format:check": "prettier --check \"src/**/*.ts\" \"test/**/*.ts\"", "check": "npm run lint && npm run test", diff --git a/src/shared/auth/__tests__/role.guard.spec.ts b/src/shared/auth/__tests__/role.guard.spec.ts index abf5a6d50a..94c5994ff3 100644 --- a/src/shared/auth/__tests__/role.guard.spec.ts +++ b/src/shared/auth/__tests__/role.guard.spec.ts @@ -1,6 +1,16 @@ import { ExecutionContext } from '@nestjs/common'; -import { hasRoleAccess, RoleGuard } from '../role.guard'; -import { UserRole } from '../user-role.enum'; + +// The clearance Set is primed by cron; mocked here so the guard's gating logic is tested in isolation +// from the cron/DB plumbing. +jest.mock('src/shared/auth/staff-kyc-clearance', () => ({ + HasStaffKycClearance: jest.fn(), +})); + +import { HasStaffKycClearance } from 'src/shared/auth/staff-kyc-clearance'; +import { hasRoleAccess, hasStaffAccess, RoleGuard, rolesSatisfying } from '../role.guard'; +import { KycGatedRoles, UserRole } from '../user-role.enum'; + +const hasStaffKycClearanceMock = HasStaffKycClearance as jest.MockedFunction; // Pins the hierarchy checks previously encoded in ad-hoc constants // (`ADMIN_ROLES.includes(role)`, `[UserRole.SUPPORT, UserRole.COMPLIANCE, ...ADMIN_ROLES].includes(role)`, @@ -71,15 +81,91 @@ describe('hasRoleAccess', () => { expect(hasRoleAccess(UserRole.ADMIN, undefined)).toBe(false); expect(hasRoleAccess(UserRole.SUPPORT, undefined)).toBe(false); }); + + // Entry roles with no `additionalRoles` entry at all (no super-roles): only the role itself matches. + it('matches only the role itself for an entry role without super-roles', () => { + expect(hasRoleAccess(UserRole.SUPER_ADMIN, UserRole.SUPER_ADMIN)).toBe(true); + expect(hasRoleAccess(UserRole.SUPER_ADMIN, UserRole.ADMIN)).toBe(false); + expect(hasRoleAccess(UserRole.CUSTODY, UserRole.ADMIN)).toBe(false); + }); +}); + +// The predicate used by the few privilege checks that live in business logic instead of a gate +// (protected KYC files, ownership-independent history access, official support replies). +describe('hasStaffAccess', () => { + afterEach(() => jest.resetAllMocks()); + + it('denies a caller whose role does not satisfy the entry role, without consulting the clearance', () => { + hasStaffKycClearanceMock.mockReturnValue(true); + + expect(hasStaffAccess(UserRole.COMPLIANCE, { role: UserRole.USER, account: 1 })).toBe(false); + expect(hasStaffKycClearanceMock).not.toHaveBeenCalled(); + }); + + it('denies an undefined jwt', () => { + expect(hasStaffAccess(UserRole.COMPLIANCE, undefined)).toBe(false); + }); + + describe.each(KycGatedRoles)('gated entry role: %s', (entryRole) => { + it('grants a cleared account', () => { + hasStaffKycClearanceMock.mockReturnValue(true); + + expect(hasStaffAccess(entryRole, { role: entryRole, account: 1 })).toBe(true); + expect(hasStaffKycClearanceMock).toHaveBeenCalledWith(1); + }); + + it('denies an uncleared account despite the correct role', () => { + hasStaffKycClearanceMock.mockReturnValue(false); + + expect(hasStaffAccess(entryRole, { role: entryRole, account: 1 })).toBe(false); + }); + }); + + it('does not require clearance for an ungated entry role', () => { + hasStaffKycClearanceMock.mockReturnValue(false); + + expect(hasStaffAccess(UserRole.USER, { role: UserRole.ADMIN, account: 1 })).toBe(true); + expect(hasStaffKycClearanceMock).not.toHaveBeenCalled(); + }); +}); + +describe('rolesSatisfying', () => { + it('covers the gated entry roles plus their super-roles', () => { + const roles = rolesSatisfying(KycGatedRoles); + + expect(roles).toEqual(expect.arrayContaining(KycGatedRoles)); + // SUPER_ADMIN satisfies every gate via the hierarchy without being listed in KycGatedRoles — + // omitting it from the clearance sync would lock super admins out of every elevated endpoint. + expect(roles).toContain(UserRole.SUPER_ADMIN); + expect(roles).not.toContain(UserRole.USER); + expect(roles).not.toContain(UserRole.ACCOUNT); + }); + + it('deduplicates roles reachable through several entry roles', () => { + expect(rolesSatisfying(KycGatedRoles)).toHaveLength(new Set(rolesSatisfying(KycGatedRoles)).size); + }); + + it('returns just the entry role when it has no super-roles', () => { + expect(rolesSatisfying([UserRole.SUPER_ADMIN])).toEqual([UserRole.SUPER_ADMIN]); + }); + + it('returns an empty list for no entry roles', () => { + expect(rolesSatisfying([])).toEqual([]); + }); }); describe('RoleGuard (multi-role access)', () => { - function contextFor(role?: UserRole): ExecutionContext { + function contextFor(role?: UserRole, account = 1): ExecutionContext { return { - switchToHttp: () => ({ getRequest: () => ({ user: role ? { role } : undefined }) }), + switchToHttp: () => ({ getRequest: () => ({ user: role ? { role, account } : undefined }) }), } as unknown as ExecutionContext; } + // Default to a cleared account so the pre-existing role-hierarchy expectations below keep testing + // the hierarchy, not the KYC gate; the gate gets its own describe block. + beforeEach(() => hasStaffKycClearanceMock.mockReturnValue(true)); + afterEach(() => jest.resetAllMocks()); + it('grants access to the single entry role and its super-roles, denies others', () => { expect(RoleGuard(UserRole.COMPLIANCE).canActivate(contextFor(UserRole.COMPLIANCE))).toBe(true); expect(RoleGuard(UserRole.COMPLIANCE).canActivate(contextFor(UserRole.ADMIN))).toBe(true); @@ -98,3 +184,62 @@ describe('RoleGuard (multi-role access)', () => { expect(RoleGuard(UserRole.COMPLIANCE, UserRole.DEBUG).canActivate(contextFor(undefined))).toBe(false); }); }); + +describe('RoleGuard (staff KYC gate on elevated endpoints)', () => { + function contextFor(role?: UserRole, account?: number): ExecutionContext { + return { + switchToHttp: () => ({ getRequest: () => ({ user: role ? { role, account } : undefined }) }), + } as unknown as ExecutionContext; + } + + afterEach(() => jest.resetAllMocks()); + + describe.each(KycGatedRoles)('entry role: %s', (entryRole) => { + it('denies the matching role when the account has no KYC clearance', () => { + hasStaffKycClearanceMock.mockReturnValue(false); + + expect(RoleGuard(entryRole).canActivate(contextFor(entryRole, 42))).toBe(false); + }); + + it('grants the matching role when the account is cleared', () => { + hasStaffKycClearanceMock.mockReturnValue(true); + + expect(RoleGuard(entryRole).canActivate(contextFor(entryRole, 42))).toBe(true); + expect(hasStaffKycClearanceMock).toHaveBeenCalledWith(42); + }); + }); + + it('gates super-roles too — an uncleared ADMIN loses every elevated endpoint', () => { + hasStaffKycClearanceMock.mockReturnValue(false); + + expect(RoleGuard(UserRole.SUPPORT).canActivate(contextFor(UserRole.ADMIN, 42))).toBe(false); + expect(RoleGuard(UserRole.COMPLIANCE, UserRole.DEBUG).canActivate(contextFor(UserRole.SUPER_ADMIN, 42))).toBe( + false, + ); + }); + + it('does not gate ordinary endpoints, even for a staff caller without clearance', () => { + hasStaffKycClearanceMock.mockReturnValue(false); + + // An admin using ordinary customer functionality is not touching an elevated endpoint. + expect(RoleGuard(UserRole.USER).canActivate(contextFor(UserRole.ADMIN, 42))).toBe(true); + expect(RoleGuard(UserRole.ACCOUNT).canActivate(contextFor(UserRole.ADMIN, 42))).toBe(true); + expect(hasStaffKycClearanceMock).not.toHaveBeenCalled(); + }); + + it('treats a gate that also admits an ungated entry role as an ordinary endpoint', () => { + hasStaffKycClearanceMock.mockReturnValue(false); + + // The gate is a property of the endpoint: one that is open to plain users is not elevated, + // regardless of which entry role the caller happens to come in through. + expect(RoleGuard(UserRole.ADMIN, UserRole.USER).canActivate(contextFor(UserRole.ADMIN, 42))).toBe(true); + }); + + it('denies an elevated endpoint when the token carries no account id', () => { + // Company tokens (and any future addressless token) have no `account` — fail closed rather than + // letting `undefined` slip through the clearance lookup. + hasStaffKycClearanceMock.mockImplementation((account) => account != null); + + expect(RoleGuard(UserRole.ADMIN).canActivate(contextFor(UserRole.ADMIN, undefined))).toBe(false); + }); +}); diff --git a/src/shared/auth/__tests__/staff-kyc-clearance.spec.ts b/src/shared/auth/__tests__/staff-kyc-clearance.spec.ts new file mode 100644 index 0000000000..73877a52cb --- /dev/null +++ b/src/shared/auth/__tests__/staff-kyc-clearance.spec.ts @@ -0,0 +1,46 @@ +import { HasStaffKycClearance, SetStaffKycClearance } from '../staff-kyc-clearance'; + +describe('staff KYC clearance', () => { + // reset the shared module-level Set so state does not leak between tests + afterEach(() => SetStaffKycClearance([])); + + it('clears an account id present in the primed list', () => { + SetStaffKycClearance([123, 456]); + + expect(HasStaffKycClearance(123)).toBe(true); + expect(HasStaffKycClearance(456)).toBe(true); + }); + + it('denies an account id not present in the list', () => { + SetStaffKycClearance([123]); + + expect(HasStaffKycClearance(999)).toBe(false); + }); + + it('denies an undefined account', () => { + SetStaffKycClearance([123]); + + expect(HasStaffKycClearance(undefined)).toBe(false); + }); + + // The inverse of the JWT denylist behaviour, and the point of the whole gate: no clearance data + // means no elevated access, never the other way round. + it('fails CLOSED before it is ever primed', () => { + expect(HasStaffKycClearance(123)).toBe(false); + }); + + it('fails CLOSED on an empty list', () => { + SetStaffKycClearance([]); + + expect(HasStaffKycClearance(123)).toBe(false); + }); + + it('revokes clearance as soon as an account drops out of the list', () => { + SetStaffKycClearance([123]); + expect(HasStaffKycClearance(123)).toBe(true); + + SetStaffKycClearance([]); + + expect(HasStaffKycClearance(123)).toBe(false); + }); +}); diff --git a/src/shared/auth/role.guard.ts b/src/shared/auth/role.guard.ts index 16c8d011ec..308d25d452 100644 --- a/src/shared/auth/role.guard.ts +++ b/src/shared/auth/role.guard.ts @@ -1,5 +1,6 @@ import { CanActivate, ExecutionContext } from '@nestjs/common'; -import { UserRole } from 'src/shared/auth/user-role.enum'; +import { HasStaffKycClearance } from 'src/shared/auth/staff-kyc-clearance'; +import { KycGatedRoles, UserRole } from 'src/shared/auth/user-role.enum'; // Role hierarchy: `additionalRoles[entryRole]` are the roles that also satisfy an // `entryRole` requirement (super-roles). Single source of truth for role checks — @@ -52,6 +53,32 @@ export function hasRoleAccess(entryRole: UserRole, userRole: UserRole | undefine return entryRole === userRole || (additionalRoles[entryRole]?.includes(userRole) ?? false); } +/** + * `hasRoleAccess` plus the staff KYC clearance that `RoleGuard` applies to elevated endpoints — for + * the handful of places that decide staff privileges in business logic rather than through a gate. + * Those sit on endpoints that are NOT role-gated (OptionalJwtAuthGuard / an ACCOUNT gate), so without + * this the KYC requirement would be bypassable: an uncleared admin would lose `/admin` yet keep + * protected KYC file downloads and ownership-independent access to any customer's history. + * + * Not needed where a privilege check merely refines behaviour behind an already-gated endpoint (e.g. + * the `isAdmin` distinctions in the support note services) — the gate has run by then. + */ +export function hasStaffAccess(entryRole: UserRole, jwt: { role?: UserRole; account?: number } | undefined): boolean { + if (!hasRoleAccess(entryRole, jwt?.role)) return false; + if (!KycGatedRoles.includes(entryRole)) return true; + return HasStaffKycClearance(jwt?.account); +} + +/** + * Every role that satisfies at least one of `entryRoles` — the entry roles themselves plus their + * super-roles per `additionalRoles`. Derived from the same map as `hasRoleAccess` so that a hierarchy + * change cannot desync the two: `StaffKycClearanceService` uses this to decide whose KYC clearance + * to track, and would otherwise silently lock out a role newly granted access to a gated endpoint. + */ +export function rolesSatisfying(entryRoles: UserRole[]): UserRole[] { + return [...new Set(entryRoles.flatMap((entryRole) => [entryRole, ...(additionalRoles[entryRole] ?? [])]))]; +} + class RoleGuardClass implements CanActivate { private readonly entryRoles: UserRole[]; @@ -60,8 +87,22 @@ class RoleGuardClass implements CanActivate { } canActivate(context: ExecutionContext): boolean { - const userRole = context.switchToHttp().getRequest().user?.role; - return this.entryRoles.some((entryRole) => hasRoleAccess(entryRole, userRole)); + const user = context.switchToHttp().getRequest().user; + if (!this.entryRoles.some((entryRole) => hasRoleAccess(entryRole, user?.role))) return false; + + // Elevated endpoint: an identified natural person must be behind the account (see KycGatedRoles). + // The gate is a property of the ENDPOINT, not of the caller, so it applies only when EVERY entry + // role is gated — a gate that also admits e.g. UserRole.USER is an ordinary endpoint that an admin + // happens to reach through the role hierarchy, and must not start demanding staff KYC. + if (this.isElevated) return HasStaffKycClearance(user?.account); + + return true; + } + + // No empty-list guard needed: `canActivate` has already returned false by then, since no role can + // satisfy an empty entry-role list. + private get isElevated(): boolean { + return this.entryRoles.every((entryRole) => KycGatedRoles.includes(entryRole)); } } diff --git a/src/shared/auth/staff-kyc-clearance.ts b/src/shared/auth/staff-kyc-clearance.ts new file mode 100644 index 0000000000..69275ff88c --- /dev/null +++ b/src/shared/auth/staff-kyc-clearance.ts @@ -0,0 +1,26 @@ +// Staff KYC clearance ALLOWlist — the inverse of the JWT denylists in ProcessService. Elevated +// endpoints (every `RoleGuard` whose entry roles are all in `KycGatedRoles`) require, on top of the +// role, that an identified natural person is behind the calling account: `kycLevel >= LEVEL_50` AND a +// non-empty `verifiedName`. `StaffKycClearanceService` derives the cleared account (user data) ids +// from the DB into the `staffKycClearance` setting; `ProcessService` primes this Set from it, so +// revoking a staff member's KYC takes effect on live tokens within one refresh interval — no +// re-login, no JWT-secret rotation. +// +// Fail-CLOSED, unlike the denylists: a not-yet-primed or empty Set denies every elevated endpoint. +// That asymmetry is deliberate — a DB or cron outage must never silently re-open admin access — and +// it is why ProcessService awaits the first prime before HTTP starts, and why a failing resync keeps +// the last known Set rather than clearing it. +// +// Deliberately its own module rather than living next to the denylists in `process.service.ts`: +// RoleGuard is imported by nearly every controller, and importing ProcessService from it would pull +// `config.ts` (and with it the whole blockchain/node-pty dependency chain) into the auth path. +let StaffKycClearedAccounts: Set = new Set(); + +export function HasStaffKycClearance(account: number | undefined): boolean { + return account != null && StaffKycClearedAccounts.has(account); +} + +// Only ProcessService should call this — the cron owns the lifecycle of the Set. +export function SetStaffKycClearance(accounts: number[]): void { + StaffKycClearedAccounts = new Set(accounts); +} diff --git a/src/shared/auth/user-role.enum.ts b/src/shared/auth/user-role.enum.ts index d1c8bc2e48..3fb67687a2 100644 --- a/src/shared/auth/user-role.enum.ts +++ b/src/shared/auth/user-role.enum.ts @@ -25,3 +25,13 @@ export enum UserRole { // must pass an independent TOTP second factor (never a mail code to the same inbox as the magic link). // Priority-ordered (highest privilege first) for mail-login role resolution. export const StaffRoles = [UserRole.COMPLIANCE, UserRole.SUPPORT, UserRole.REALUNIT]; + +// Entry roles that mark an endpoint as elevated: reaching it requires an identified natural person +// behind the account, on top of the role itself. `RoleGuard` therefore demands staff KYC clearance +// (`kycLevel >= LEVEL_50` AND a non-empty `verifiedName`, see `HasStaffKycClearance`) whenever every +// entry role of a gate is listed here. Distinct from `StaffRoles` above, which is about mail-login +// role resolution — this list is about endpoint sensitivity and also covers ADMIN and DEBUG. +// +// Not listed, deliberately: BANKING_BOT and CUSTODY are non-staff entry roles and stay ungated, so a +// cleared-role holder reaching those endpoints via the `additionalRoles` hierarchy is not KYC-gated. +export const KycGatedRoles = [UserRole.ADMIN, UserRole.DEBUG, UserRole.COMPLIANCE, UserRole.SUPPORT, UserRole.REALUNIT]; diff --git a/src/shared/models/setting/__tests__/setting.service.spec.ts b/src/shared/models/setting/__tests__/setting.service.spec.ts index 43a5a44d63..e313a887a9 100644 --- a/src/shared/models/setting/__tests__/setting.service.spec.ts +++ b/src/shared/models/setting/__tests__/setting.service.spec.ts @@ -1,8 +1,9 @@ +import { ForbiddenException } from '@nestjs/common'; import { createMock } from '@golevelup/ts-jest'; import { Test, TestingModule } from '@nestjs/testing'; import { Setting } from '../setting.entity'; import { SettingRepository } from '../setting.repository'; -import { SettingService } from '../setting.service'; +import { SettingService, SystemManagedSettings } from '../setting.service'; describe('SettingService', () => { let service: SettingService; @@ -72,4 +73,62 @@ describe('SettingService', () => { await expect(service.getDeniedJwtAccounts()).resolves.toEqual([5, 6]); }); }); + + describe('set', () => { + it('writes an ordinary setting under the given key', async () => { + settingRepo.findOneBy.mockResolvedValue(null); + settingRepo.create.mockImplementation((data) => Object.assign(new Setting(), data)); + + await service.set('someOpsFlag', 'true'); + + // Pins key and value, not just that something was written: a swapped argument pair would + // otherwise pass, and this is the only place the pair is checked. + expect(settingRepo.save).toHaveBeenCalledWith(expect.objectContaining({ key: 'someOpsFlag', value: 'true' })); + }); + + // `staffKycClearance` decides who reaches every elevated endpoint and is derived from KYC data by a + // sync job. A manual write through the generic setter would grant elevated access to accounts that + // never passed KYC, and would stay live until the next sync run overwrote it. The check sits here + // rather than in the controller so every caller of `set` is covered, not just the HTTP route. + describe.each(SystemManagedSettings)('system-managed setting %s', (key) => { + it('is rejected, without writing', async () => { + await expect(service.set(key, '[1,2,3]')).rejects.toBeInstanceOf(ForbiddenException); + + expect(settingRepo.save).not.toHaveBeenCalled(); + }); + }); + + it('lists staffKycClearance as system-managed', () => { + expect(SystemManagedSettings).toContain('staffKycClearance'); + }); + + // The sync path writes through `setObj`, which goes to the repository directly — blocking it would + // freeze the allowlist at whatever it happened to contain. + it('still allows the sync path to write the clearance through setObj', async () => { + await service.setObj('staffKycClearance', [1, 2]); + + expect(settingRepo.save).toHaveBeenCalled(); + }); + }); + + describe('getStaffKycClearance', () => { + it('returns the cleared account ids', async () => { + mockSettings({ staffKycClearance: [1, 2] }); + + await expect(service.getStaffKycClearance()).resolves.toEqual([1, 2]); + }); + + it('coerces string ids to numbers', async () => { + mockSettings({ staffKycClearance: ['1', '2'] }); + + await expect(service.getStaffKycClearance()).resolves.toEqual([1, 2]); + }); + + // Fail-closed: a missing setting must read as "nobody is cleared", never as "no restriction". + it('returns an empty array when the setting is missing', async () => { + mockSettings({}); + + await expect(service.getStaffKycClearance()).resolves.toEqual([]); + }); + }); }); diff --git a/src/shared/models/setting/setting.service.ts b/src/shared/models/setting/setting.service.ts index 680ae01222..d4219c9a68 100644 --- a/src/shared/models/setting/setting.service.ts +++ b/src/shared/models/setting/setting.service.ts @@ -1,4 +1,4 @@ -import { BadRequestException, Injectable } from '@nestjs/common'; +import { BadRequestException, ForbiddenException, Injectable } from '@nestjs/common'; import { plainToInstance } from 'class-transformer'; import { validate } from 'class-validator'; import { Process } from 'src/shared/services/process.service'; @@ -8,6 +8,12 @@ import { isArraySchema, isPrimitiveSchema, SettingSchema, SettingSchemaRegistry import { Setting } from './setting.entity'; import { SettingRepository } from './setting.repository'; +// Settings whose value is derived by a sync job, never by an operator. The generic `PUT /setting/:key` +// route rejects them: for `staffKycClearance` a manual write would hand elevated access to accounts that +// never passed KYC — the very thing the gate exists to prevent — and would stay live until the next sync +// overwrites it. The sync path itself writes through `setObj` and is unaffected. +export const SystemManagedSettings = ['staffKycClearance']; + @Injectable() export class SettingService { constructor(private readonly settingRepo: SettingRepository) {} @@ -27,6 +33,11 @@ export class SettingService { } async set(key: string, value: string): Promise { + // Sync-owned settings are not writable through the generic setter — see SystemManagedSettings. + // `setObj` (the sync path) writes to the repository directly and is deliberately unaffected. + if (SystemManagedSettings.includes(key)) + throw new ForbiddenException(`Setting ${key} is maintained by the system and cannot be set manually`); + await this.validateSettingValue(key, value); const entity = (await this.settingRepo.findOneBy({ key })) ?? this.settingRepo.create({ key }); @@ -131,6 +142,15 @@ export class SettingService { return [...new Set([...manual, ...auto].map(Number))]; } + // Account (user data) ids cleared for elevated endpoints, maintained by StaffKycClearanceService. + // No manual-override counterpart on purpose: the clearance is a KYC fact, not an ops decision — an + // editable override would be a way to hand out admin access without the identification behind it. + // The generic `PUT /setting/:key` route would be exactly such an override, which is why the key is + // listed in `SystemManagedSettings` above and rejected by `set`. + async getStaffKycClearance(): Promise { + return this.getObj<(string | number)[]>('staffKycClearance', []).then((list) => list.map(Number)); + } + async getCustomBalanceSettings(): Promise<{ addresses: string[]; assets: string[] }> { const [addresses, assets] = await Promise.all([ this.getObjCached('customBalanceAddresses', []), diff --git a/src/shared/services/__tests__/process.service.spec.ts b/src/shared/services/__tests__/process.service.spec.ts index daf3feb509..45230e301a 100644 --- a/src/shared/services/__tests__/process.service.spec.ts +++ b/src/shared/services/__tests__/process.service.spec.ts @@ -1,3 +1,4 @@ +import { HasStaffKycClearance, SetStaffKycClearance } from 'src/shared/auth/staff-kyc-clearance'; import { SettingService } from 'src/shared/models/setting/setting.service'; import { IsJwtAccountDenied, ProcessService } from 'src/shared/services/process.service'; @@ -47,3 +48,38 @@ describe('ProcessService JWT account denylist', () => { expect(IsJwtAccountDenied(123)).toBe(false); }); }); + +describe('ProcessService staff KYC clearance priming', () => { + let settingService: jest.Mocked; + let service: ProcessService; + + beforeEach(() => { + settingService = { + getStaffKycClearance: jest.fn().mockResolvedValue([]), + } as unknown as jest.Mocked; + + service = new ProcessService(settingService); + }); + + afterEach(() => SetStaffKycClearance([])); + + it('primes the in-memory clearance Set from the setting', async () => { + settingService.getStaffKycClearance.mockResolvedValue([123, 456]); + + await service.resyncStaffKycClearance(); + + expect(HasStaffKycClearance(123)).toBe(true); + expect(HasStaffKycClearance(456)).toBe(true); + expect(HasStaffKycClearance(999)).toBe(false); + }); + + it('drops a revoked account on the next resync', async () => { + settingService.getStaffKycClearance.mockResolvedValue([123]); + await service.resyncStaffKycClearance(); + + settingService.getStaffKycClearance.mockResolvedValue([]); + await service.resyncStaffKycClearance(); + + expect(HasStaffKycClearance(123)).toBe(false); + }); +}); diff --git a/src/shared/services/process.service.ts b/src/shared/services/process.service.ts index 97d79a09b5..4a9f1ba2e5 100644 --- a/src/shared/services/process.service.ts +++ b/src/shared/services/process.service.ts @@ -1,6 +1,7 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; import { Config } from 'src/config/config'; +import { SetStaffKycClearance } from 'src/shared/auth/staff-kyc-clearance'; import { SettingService } from '../models/setting/setting.service'; import { DfxCron } from '../utils/cron'; @@ -174,6 +175,9 @@ export class ProcessService implements OnModuleInit { // is fail-closed by sentinel await this.resyncDeniedJwtAddresses(); await this.resyncDeniedJwtAccounts(); + // await as well, but for the opposite reason: the staff clearance Set is fail-closed, so serving + // HTTP before it is primed would deny every elevated endpoint instead of over-granting. + await this.resyncStaffKycClearance(); } @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800 }) @@ -198,6 +202,13 @@ export class ProcessService implements OnModuleInit { DeniedJwtAccounts = new Set(list); } + // Primes the fail-closed staff clearance allowlist — see `staff-kyc-clearance.ts` for the semantics. + @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800 }) + async resyncStaffKycClearance(): Promise { + const list = await this.settingService.getStaffKycClearance(); + SetStaffKycClearance(list); + } + public async setSafetyModeActive(active: boolean): Promise { this.safetyModeInactive = DisabledProcess(Process.SAFETY_MODE) ? true : !active; await this.resyncDisabledProcesses(); diff --git a/src/subdomains/core/history/services/__tests__/history-access.service.spec.ts b/src/subdomains/core/history/services/__tests__/history-access.service.spec.ts index a10c60a79f..71627fe124 100644 --- a/src/subdomains/core/history/services/__tests__/history-access.service.spec.ts +++ b/src/subdomains/core/history/services/__tests__/history-access.service.spec.ts @@ -1,6 +1,7 @@ import { createMock } from '@golevelup/ts-jest'; import { ForbiddenException, UnauthorizedException } from '@nestjs/common'; import { JwtPayload } from 'src/shared/auth/jwt-payload.interface'; +import { SetStaffKycClearance } from 'src/shared/auth/staff-kyc-clearance'; import { UserRole } from 'src/shared/auth/user-role.enum'; import { UserData } from 'src/subdomains/generic/user/models/user-data/user-data.entity'; import { UserDataService } from 'src/subdomains/generic/user/models/user-data/user-data.service'; @@ -164,6 +165,11 @@ describe('HistoryAccessService', () => { const ownerJwt: JwtPayload = { role: UserRole.USER, ip: '1.1.1.1', account: 1, user: 10, address: '0xAAA' }; const otherJwt: JwtPayload = { role: UserRole.USER, ip: '1.1.1.1', account: 2, user: 20, address: '0xCCC' }; + // Staff full access now additionally requires KYC clearance for the calling account; account 99 is + // the cleared staff account used by the staff cases below. + beforeEach(() => SetStaffKycClearance([99])); + afterEach(() => SetStaffKycClearance([])); + it('denies full view without JWT', () => { const tx = { userData: { id: 1 } } as Transaction; expect(service.canViewFullTransaction(undefined, tx)).toBe(false); @@ -206,6 +212,27 @@ describe('HistoryAccessService', () => { expect(service.canViewFullTransaction(staff, tx)).toBe(true); }); + it.each([UserRole.SUPPORT, UserRole.COMPLIANCE, UserRole.ADMIN])( + 'denies %s staff full access without KYC clearance', + (role) => { + SetStaffKycClearance([]); + const staff: JwtPayload = { role, ip: '1.1.1.1', account: 99 }; + const tx = { userData: { id: 1 } } as Transaction; + + // These routes are OptionalJwtAuthGuard-only, so this predicate is the only place the staff KYC + // gate can apply — without it, an uncleared admin keeps blanket access to every customer's tx. + expect(service.canViewFullTransaction(staff, tx)).toBe(false); + }, + ); + + it('still allows an uncleared staff member to view their OWN transaction', () => { + SetStaffKycClearance([]); + const staff: JwtPayload = { role: UserRole.ADMIN, ip: '1.1.1.1', account: 99 }; + const tx = { userData: { id: 99 } } as Transaction; + + expect(service.canViewFullTransaction(staff, tx)).toBe(true); + }); + it('denies REALUNIT ownership-independent full access (isolated external tenant, not DFX staff)', () => { const tenantStaff: JwtPayload = { role: UserRole.REALUNIT, ip: '1.1.1.1', account: 99 }; const tx = { userData: { id: 1 } } as Transaction; // not owned by the RealUnit tenant account diff --git a/src/subdomains/core/history/services/history-access.service.ts b/src/subdomains/core/history/services/history-access.service.ts index a55f976950..b8ec9325f1 100644 --- a/src/subdomains/core/history/services/history-access.service.ts +++ b/src/subdomains/core/history/services/history-access.service.ts @@ -1,6 +1,6 @@ import { ForbiddenException, Injectable, UnauthorizedException } from '@nestjs/common'; import { JwtPayload } from 'src/shared/auth/jwt-payload.interface'; -import { hasRoleAccess } from 'src/shared/auth/role.guard'; +import { hasStaffAccess } from 'src/shared/auth/role.guard'; import { UserRole } from 'src/shared/auth/user-role.enum'; import { UserData } from 'src/subdomains/generic/user/models/user-data/user-data.entity'; import { UserDataService } from 'src/subdomains/generic/user/models/user-data/user-data.service'; @@ -55,7 +55,7 @@ export class HistoryAccessService { */ canViewFullTransaction(jwt: JwtPayload | undefined, tx: Transaction | TransactionRequest | undefined): boolean { if (!jwt?.role) return false; - if (this.isStaffFullAccess(jwt.role)) return true; + if (this.isStaffFullAccess(jwt)) return true; return this.isOwner(jwt, tx); } @@ -65,12 +65,16 @@ export class HistoryAccessService { return accountId != null && accountId === jwt.account; } - private isStaffFullAccess(role: UserRole): boolean { - // DFX staff only: the SUPPORT hierarchy (COMPLIANCE / ADMIN / SUPER_ADMIN via hasRoleAccess). REALUNIT is an + private isStaffFullAccess(jwt: JwtPayload): boolean { + // DFX staff only: the SUPPORT hierarchy (COMPLIANCE / ADMIN / SUPER_ADMIN via hasStaffAccess). REALUNIT is an // isolated external tenant, NOT DFX staff — granting it ownership-independent full access here would leak every // customer's private banking/compliance data across the tenant boundary that its own routes scope via // RealUnitScopeService. RealUnit access to a customer's transaction must stay customer-scoped, never blanket. - return hasRoleAccess(UserRole.SUPPORT, role); + // + // `hasStaffAccess`, not `hasRoleAccess`: these routes are OptionalJwtAuthGuard-only, so no RoleGuard has + // applied the staff KYC gate. Ownership-independent access to every customer's history is exactly the + // privilege the gate exists for. + return hasStaffAccess(UserRole.SUPPORT, jwt); } private accountIdOf(tx: Transaction | TransactionRequest): number | undefined { diff --git a/src/subdomains/generic/kyc/services/__tests__/kyc.service.spec.ts b/src/subdomains/generic/kyc/services/__tests__/kyc.service.spec.ts index 73d6fdacdc..582086c6aa 100644 --- a/src/subdomains/generic/kyc/services/__tests__/kyc.service.spec.ts +++ b/src/subdomains/generic/kyc/services/__tests__/kyc.service.spec.ts @@ -3,6 +3,7 @@ import { ForbiddenException } from '@nestjs/common'; import { Configuration, ConfigService } from 'src/config/config'; import { BlobContent } from 'src/integration/infrastructure/storage/storage.service'; import { JwtPayload } from 'src/shared/auth/jwt-payload.interface'; +import { SetStaffKycClearance } from 'src/shared/auth/staff-kyc-clearance'; import { UserRole } from 'src/shared/auth/user-role.enum'; import { createCustomCountry } from 'src/shared/models/country/__mocks__/country.entity.mock'; import { Country } from 'src/shared/models/country/country.entity'; @@ -120,6 +121,10 @@ describe('KycService getFileByUid protected-file access', () => { }); beforeEach(() => { + // Protected-file access now additionally requires staff KYC clearance for the calling account + // (account 1 in `jwtFor`); the uncleared case has its own test below. + SetStaffKycClearance([1]); + kycFileService = createMock(); documentService = createMock(); tfaService = { check: jest.fn() }; @@ -164,6 +169,28 @@ describe('KycService getFileByUid protected-file access', () => { expect(documentService.downloadFile).not.toHaveBeenCalled(); }); + // This route is OptionalJwtAuthGuard-only, so no RoleGuard has applied the staff KYC gate — the role + // check inside the service is the only thing standing between an uncleared admin and the most + // sensitive sink in the API. + describe.each([UserRole.SUPER_ADMIN, UserRole.ADMIN, UserRole.COMPLIANCE])('%s without KYC clearance', (role) => { + it('is forbidden from a protected file, without downloading', async () => { + SetStaffKycClearance([]); + kycFileService.getKycFile.mockResolvedValue(kycFile()); + + await expect(service.getFileByUid('FILE-UID', jwtFor(role), ip)).rejects.toBeInstanceOf(ForbiddenException); + expect(documentService.downloadFile).not.toHaveBeenCalled(); + }); + + it('still serves a non-protected file', async () => { + SetStaffKycClearance([]); + kycFileService.getKycFile.mockResolvedValue(kycFile({ protected: false })); + + const dto = await service.getFileByUid('FILE-UID', jwtFor(role), ip); + + expect(dto.uid).toBe('FILE-UID'); + }); + }); + // a blocked account keeps its JWT role until expiry, so the status check must still deny access describe.each<[UserRole, Partial]>([ [UserRole.ADMIN, { accountStatus: UserDataStatus.BLOCKED }], diff --git a/src/subdomains/generic/kyc/services/kyc.service.ts b/src/subdomains/generic/kyc/services/kyc.service.ts index 5f6cb2dda9..dc68c325bc 100644 --- a/src/subdomains/generic/kyc/services/kyc.service.ts +++ b/src/subdomains/generic/kyc/services/kyc.service.ts @@ -9,7 +9,7 @@ import { import { CronExpression } from '@nestjs/schedule'; import { Config } from 'src/config/config'; import { JwtPayload } from 'src/shared/auth/jwt-payload.interface'; -import { hasRoleAccess } from 'src/shared/auth/role.guard'; +import { hasStaffAccess } from 'src/shared/auth/role.guard'; import { isUserActive } from 'src/shared/auth/user-active.guard'; import { UserRole } from 'src/shared/auth/user-role.enum'; import { Country } from 'src/shared/models/country/country.entity'; @@ -473,8 +473,10 @@ export class KycService { if (!kycFile) throw new NotFoundException('KYC file not found'); if (kycFile.protected) { - if (!hasRoleAccess(UserRole.COMPLIANCE, jwt?.role)) - throw new ForbiddenException('Requires admin or compliance role'); + // `hasStaffAccess`, not `hasRoleAccess`: this route is OptionalJwtAuthGuard-only, so no RoleGuard has + // applied the staff KYC gate. Protected KYC files are the most sensitive sink in the API — an + // uncleared compliance/admin account must not reach them just because the endpoint is not role-gated. + if (!hasStaffAccess(UserRole.COMPLIANCE, jwt)) throw new ForbiddenException('Requires admin or compliance role'); if (!jwt || !isUserActive(jwt)) throw new ForbiddenException('User is not active'); // Mail-origin staff sessions (tfaRequired) must complete STRICT 2FA before downloading protected KYC diff --git a/src/subdomains/generic/user/models/user/__tests__/staff-kyc-clearance.pg.spec.ts b/src/subdomains/generic/user/models/user/__tests__/staff-kyc-clearance.pg.spec.ts new file mode 100644 index 0000000000..79ad9d2f4a --- /dev/null +++ b/src/subdomains/generic/user/models/user/__tests__/staff-kyc-clearance.pg.spec.ts @@ -0,0 +1,118 @@ +import { DataSource } from 'typeorm'; +import { BlankChars, nonBlankPredicate } from '../staff-kyc-clearance.service'; + +// The clearance query decides who reaches every elevated endpoint, and its verifiedName condition is raw +// SQL: a mocked repository never executes it, so the unit suite cannot tell a correct predicate from one +// that silently clears blank names. This suite runs the exact fragment against a real Postgres and pins +// it to the semantics it replaced — `String.prototype.trim()`. +// +// Skipped unless a connection string is provided, matching the migration suites in this repo; CI runs it +// against its throwaway Postgres service. +const PG_URL = process.env.MIGRATION_TEST_PG; +const describeDb = PG_URL ? describe : describe.skip; + +const SCHEMA = 'staff_kyc_clearance_spec'; + +// Blank in JS, and every one of them a plausible copy-paste artefact in an identity field. Bare +// Postgres `TRIM()` strips only U+0020, so all but the plain-space cases would survive it. +const BLANK_NAMES = [ + '', + '\u0020', + '\u0020\u0020\u0020', + '\u0009', // tab + '\u000a', // line feed + '\u000d\u000a', // CRLF + '\u000b', // vertical tab + '\u000c', // form feed + '\u00a0', // non-breaking space + '\u1680', // ogham space mark + '\u2003', // em space + '\u202f', // narrow no-break space + '\u205f', // medium mathematical space + '\u3000', // ideographic space + '\ufeff', // zero-width no-break space + '\u0020\u0009\u00a0\u3000\u0020', // mixed +]; + +// Every character the JS runtime treats as blank, derived from the runtime rather than from BlankChars: +// fixtures generated from the constant under test would shrink along with it and could never catch a +// character dropped from it. +const JS_BLANK_CHARS = Array.from({ length: 0x10000 }, (_, code) => String.fromCharCode(code)).filter( + (char) => char.trim() === '', +); + +// Real identifications, including ones padded with the same characters — padding must never disqualify. +const REAL_NAMES = [ + 'Alice Example', + '\u0020Alice Example\u0020', + '\u0009Alice Example\u000a', + '\u00a0Alice Example\u3000', + 'X', +]; + +describeDb('staff KYC clearance verifiedName predicate (real Postgres)', () => { + let dataSource: DataSource; + + beforeAll(async () => { + dataSource = new DataSource({ type: 'postgres', url: PG_URL }); + await dataSource.initialize(); + await dataSource.query(`DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE`); + await dataSource.query(`CREATE SCHEMA ${SCHEMA}`); + // Quoted camelCase column, exactly as the migrations create it — an unquoted identifier would be + // folded to lowercase by Postgres and the predicate would fail at runtime rather than in review. + await dataSource.query(`CREATE TABLE ${SCHEMA}.user_data (id serial PRIMARY KEY, "verifiedName" varchar)`); + }); + + afterAll(async () => { + if (!dataSource?.isInitialized) return; + await dataSource.query(`DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE`); + await dataSource.destroy(); + }); + + async function selectMatching(names: (string | null)[]): Promise<(string | null)[]> { + await dataSource.query(`TRUNCATE ${SCHEMA}.user_data`); + for (const name of names) { + await dataSource.query(`INSERT INTO ${SCHEMA}.user_data ("verifiedName") VALUES ($1)`, [name]); + } + + // $1 stands in for the named :blankChars parameter TypeORM binds; the fragment itself is verbatim. + const predicate = nonBlankPredicate('"verifiedName"').replace(':blankChars', '$1'); + const rows = await dataSource.query( + `SELECT "verifiedName" FROM ${SCHEMA}.user_data WHERE ${predicate} ORDER BY id`, + [BlankChars], + ); + + return rows.map((row: { verifiedName: string | null }) => row.verifiedName); + } + + it('excludes NULL', async () => { + await expect(selectMatching([null])).resolves.toEqual([]); + }); + + it.each(BLANK_NAMES)('excludes the blank name %j', async (name) => { + await expect(selectMatching([name])).resolves.toEqual([]); + }); + + it.each(REAL_NAMES)('keeps the real name %j', async (name) => { + await expect(selectMatching([name])).resolves.toEqual([name]); + }); + + // Guards the character set itself: drop a character from BlankChars and a name consisting of it starts + // clearing an account. The named cases above stay for readability — this one is the exhaustive check. + it('excludes every character the JS runtime treats as blank', async () => { + expect(JS_BLANK_CHARS.length).toBeGreaterThan(20); // sanity: the derivation actually found them + + await expect(selectMatching(JS_BLANK_CHARS)).resolves.toEqual([]); + }); + + // The property that matters: the SQL predicate and the JS check it replaced must agree on every input. + // A disagreement here is either a locked-out staff member or a cleared account with no identification. + it('agrees with String.prototype.trim() on every case', async () => { + const all = [null, ...BLANK_NAMES, ...JS_BLANK_CHARS, ...REAL_NAMES]; + + const fromSql = await selectMatching(all); + const fromJs = all.filter((name) => name?.trim()); + + expect(fromSql).toEqual(fromJs); + }); +}); diff --git a/src/subdomains/generic/user/models/user/__tests__/staff-kyc-clearance.service.spec.ts b/src/subdomains/generic/user/models/user/__tests__/staff-kyc-clearance.service.spec.ts new file mode 100644 index 0000000000..6b95fabb01 --- /dev/null +++ b/src/subdomains/generic/user/models/user/__tests__/staff-kyc-clearance.service.spec.ts @@ -0,0 +1,103 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { SettingService } from 'src/shared/models/setting/setting.service'; +import { StaffKycClearanceService } from '../staff-kyc-clearance.service'; +import { UserRepository } from '../user.repository'; + +describe('StaffKycClearanceService', () => { + let service: StaffKycClearanceService; + let userRepo: UserRepository; + let settingService: SettingService; + + function setup(users: unknown[]): void { + jest.spyOn(userRepo, 'find').mockResolvedValue(users as never); + } + + // Rows as the DB returns them: the role / kycLevel / verifiedName filtering has already happened in + // SQL, so a returned row is by definition a cleared one. + function staffUser(accountId: number): unknown { + return { id: accountId * 10, userData: { id: accountId } }; + } + + beforeEach(async () => { + userRepo = { find: jest.fn() } as unknown as UserRepository; + settingService = { setObj: jest.fn() } as unknown as SettingService; + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + StaffKycClearanceService, + { provide: UserRepository, useValue: userRepo }, + { provide: SettingService, useValue: settingService }, + ], + }).compile(); + + service = module.get(StaffKycClearanceService); + }); + + afterEach(() => jest.resetAllMocks()); + + it('writes the cleared account ids to the staffKycClearance setting', async () => { + setup([staffUser(11), staffUser(12)]); + + await service.syncStaffKycClearance(); + + expect(settingService.setObj).toHaveBeenCalledWith('staffKycClearance', [11, 12]); + }); + + // Accounts without a usable verifiedName (NULL, empty or whitespace-only) are excluded by the SQL + // predicate, not in JS — so the assertion has to be on the query. `TRIM(...) <> ''` also drops NULL, + // because the comparison yields NULL rather than true. + it('excludes names that are NULL, empty or whitespace-only via a SQL predicate', async () => { + setup([]); + + await service.syncStaffKycClearance(); + + const verifiedName = (userRepo.find as jest.Mock).mock.calls[0][0].where.userData.verifiedName; + expect(verifiedName.type).toBe('raw'); + // The alias must be interpolated verbatim: TypeORM passes it in already quoted, and on Postgres an + // unquoted camelCase identifier would be folded to lowercase and blow up at runtime. + expect(verifiedName.getSql('"UserData"."verifiedName"')).toBe( + `BTRIM("UserData"."verifiedName", :blankChars) <> ''`, + ); + // Bare TRIM() would strip ASCII space only; the bound set must carry the characters that make the + // predicate agree with String.prototype.trim(). Whether it actually does is proven against a real + // database in staff-kyc-clearance.pg.spec.ts — a mocked repository never runs the fragment. + expect(verifiedName.objectLiteralParameters.blankChars).toContain('\u0009'); + expect(verifiedName.objectLiteralParameters.blankChars).toContain('\u00a0'); + expect(verifiedName.objectLiteralParameters.blankChars).toContain('\u3000'); + }); + + it('deduplicates accounts backing several staff users', async () => { + // One person can hold multiple staff wallets pointing at the same user data. + setup([staffUser(11), staffUser(11)]); + + await service.syncStaffKycClearance(); + + expect(settingService.setObj).toHaveBeenCalledWith('staffKycClearance', [11]); + }); + + it('writes an empty list when nobody qualifies — the gate is fail-closed', async () => { + setup([]); + + await service.syncStaffKycClearance(); + + expect(settingService.setObj).toHaveBeenCalledWith('staffKycClearance', []); + }); + + it('queries only staff roles and kycLevel >= 50', async () => { + setup([]); + + await service.syncStaffKycClearance(); + + const where = (userRepo.find as jest.Mock).mock.calls[0][0].where; + expect(where.role._value).toEqual(expect.arrayContaining(['Admin', 'SuperAdmin', 'Debug', 'RealUnit'])); + expect(where.role._value).not.toContain('User'); + expect(where.userData.kycLevel._value).toBe(50); + }); + + it('does not swallow a repository failure — a failed sync must keep the last known Set', async () => { + jest.spyOn(userRepo, 'find').mockRejectedValue(new Error('db down')); + + await expect(service.syncStaffKycClearance()).rejects.toThrow('db down'); + expect(settingService.setObj).not.toHaveBeenCalled(); + }); +}); diff --git a/src/subdomains/generic/user/models/user/staff-kyc-clearance.service.ts b/src/subdomains/generic/user/models/user/staff-kyc-clearance.service.ts new file mode 100644 index 0000000000..2a3aa9feb6 --- /dev/null +++ b/src/subdomains/generic/user/models/user/staff-kyc-clearance.service.ts @@ -0,0 +1,74 @@ +import { Injectable } from '@nestjs/common'; +import { CronExpression } from '@nestjs/schedule'; +import { rolesSatisfying } from 'src/shared/auth/role.guard'; +import { KycGatedRoles } from 'src/shared/auth/user-role.enum'; +import { SettingService } from 'src/shared/models/setting/setting.service'; +import { DfxCron } from 'src/shared/utils/cron'; +import { In, MoreThanOrEqual, Raw } from 'typeorm'; +import { KycLevel } from '../user-data/user-data.enum'; +import { UserRepository } from './user.repository'; + +// Roles that can reach a KYC-gated endpoint: the gated entry roles plus their super-roles (e.g. +// SUPER_ADMIN, which satisfies every gate but is not itself listed in KycGatedRoles). Derived, not +// hand-written — a role added to the hierarchy must not silently fall out of the clearance sync and +// lose access. +const ClearanceRelevantRoles = rolesSatisfying(KycGatedRoles); + +// Every character `String.prototype.trim()` strips (ECMAScript WhiteSpace + LineTerminator). Postgres' +// bare `TRIM(x)` removes ASCII space ONLY, so a name of a single tab or a non-breaking space would pass +// a `TRIM(x) <> ''` test and clear an account that carries no identification at all. The set is spelled +// out rather than left to `[[:space:]]`, whose meaning depends on the database locale. +// +// U+200B (zero width space) is deliberately NOT in here: `trim()` does not strip it either, so adding it +// would make this predicate stricter than the check it replaced. Whether a name made of invisible +// characters that `trim()` ignores should count as identification is a question about how verifiedName is +// written, not about this gate. +export const BlankChars = + '\u0009\u000a\u000b\u000c\u000d\u0020\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007' + + '\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff'; + +// Exported so the Postgres suite can execute this exact predicate against a real database — a mocked +// repository never runs the fragment, and the ASCII-only default of `TRIM` is invisible without one. +export function nonBlankPredicate(alias: string): string { + return `BTRIM(${alias}, :blankChars) <> ''`; +} + +// Maintains the `staffKycClearance` setting: the account (user data) ids allowed onto elevated +// endpoints. `ProcessService` primes the in-memory Set from it and `RoleGuard` enforces it — see +// `HasStaffKycClearance` for the fail-closed semantics. +// +// Mirrors JwtRevocationSyncService: the cron lives in the user domain (which owns User/UserData) to +// keep the shared ProcessService/SettingService free of subdomain dependencies. Self-healing in both +// directions — losing kycLevel, losing verifiedName, or losing the staff role drops the account out of +// the query and thus out of the setting on the next run. +@Injectable() +export class StaffKycClearanceService { + constructor( + private readonly userRepo: UserRepository, + private readonly settingService: SettingService, + ) {} + + // Every minute, matching JwtRevocationSyncService: revoking elevated access promptly is a security + // requirement and warrants the same exception to the "prefer 15min" cron guideline. + @DfxCron(CronExpression.EVERY_MINUTE, { timeout: 1800 }) + async syncStaffKycClearance(): Promise { + const staffUsers = await this.userRepo.find({ + select: { id: true, userData: { id: true } }, + where: { + role: In(ClearanceRelevantRoles), + userData: { + kycLevel: MoreThanOrEqual(KycLevel.LEVEL_50), + // `verifiedName IS NOT NULL` is the stated rule, but an empty or blank name carries no + // identification either — the predicate covers both, and NULL drops out on its own because the + // comparison yields NULL. See BlankChars for why the character set is explicit. + verifiedName: Raw(nonBlankPredicate, { blankChars: BlankChars }), + }, + }, + relations: { userData: true }, + }); + + const clearedAccounts = staffUsers.map((user) => user.userData.id); + + await this.settingService.setObj('staffKycClearance', [...new Set(clearedAccounts)]); + } +} diff --git a/src/subdomains/generic/user/user.module.ts b/src/subdomains/generic/user/user.module.ts index 5fce80a978..d6d0860a7c 100644 --- a/src/subdomains/generic/user/user.module.ts +++ b/src/subdomains/generic/user/user.module.ts @@ -48,6 +48,7 @@ import { JwtRevocationSyncService } from './models/user-data/jwt-revocation-sync import { UserDataJobService } from './models/user-data/user-data-job.service'; import { UserDataNotificationService } from './models/user-data/user-data-notification.service'; import { UserData } from './models/user-data/user-data.entity'; +import { StaffKycClearanceService } from './models/user/staff-kyc-clearance.service'; import { UserJobService } from './models/user/user-job.service'; import { UserController, UserV2Controller } from './models/user/user.controller'; import { User } from './models/user/user.entity'; @@ -126,6 +127,7 @@ import { WebhookService } from './services/webhook/webhook.service'; OrganizationRepository, UserDataJobService, JwtRevocationSyncService, + StaffKycClearanceService, UserJobService, RecommendationRepository, RecommendationService, diff --git a/src/subdomains/supporting/support-issue/__tests__/support-issue.controller.spec.ts b/src/subdomains/supporting/support-issue/__tests__/support-issue.controller.spec.ts index ddb38bddf6..f95f40e168 100644 --- a/src/subdomains/supporting/support-issue/__tests__/support-issue.controller.spec.ts +++ b/src/subdomains/supporting/support-issue/__tests__/support-issue.controller.spec.ts @@ -1,6 +1,7 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest'; import { ModuleRef } from '@nestjs/core'; import { JwtPayload } from 'src/shared/auth/jwt-payload.interface'; +import { SetStaffKycClearance } from 'src/shared/auth/staff-kyc-clearance'; import { UserRole } from 'src/shared/auth/user-role.enum'; import { CreateSupportMessageDto } from '../dto/create-support-message.dto'; import { SupportEscalationService } from '../services/support-escalation.service'; @@ -28,6 +29,10 @@ describe('SupportIssueController.createSupportMessage routing', () => { const ip = '1.2.3.4'; beforeEach(() => { + // Staff routing now additionally requires KYC clearance for the calling account (account 7 in the + // staff cases below); the uncleared case has its own test. + SetStaffKycClearance([7]); + service = createMock(); tfaService = { check: jest.fn() }; moduleRef = createMock(); @@ -47,6 +52,22 @@ describe('SupportIssueController.createSupportMessage routing', () => { }, ); + // This route is OptionalJwtAuthGuard-only, so the staff KYC gate has to be applied inline. An + // uncleared staff account falls through to the customer path rather than posting an official reply. + describe.each([UserRole.SUPPORT, UserRole.COMPLIANCE, UserRole.ADMIN, UserRole.SUPER_ADMIN])( + 'staff role %s without KYC clearance', + (role) => { + it('falls through to createMessage instead of posting an official reply', async () => { + SetStaffKycClearance([]); + + await controller.createSupportMessage({ role, account: 7 } as JwtPayload, '42', dto, ip); + + expect(service.createMessageSupport).not.toHaveBeenCalled(); + expect(service.createMessage).toHaveBeenCalledWith('42', dto, 7); + }); + }, + ); + it('routes a regular user message to createMessage', async () => { await controller.createSupportMessage({ role: UserRole.USER, account: 7 } as JwtPayload, '42', dto, ip); diff --git a/src/subdomains/supporting/support-issue/support-issue.controller.ts b/src/subdomains/supporting/support-issue/support-issue.controller.ts index a36d64077c..9a36160e8b 100644 --- a/src/subdomains/supporting/support-issue/support-issue.controller.ts +++ b/src/subdomains/supporting/support-issue/support-issue.controller.ts @@ -7,7 +7,7 @@ import { GetJwt } from 'src/shared/auth/get-jwt.decorator'; import { JwtPayload } from 'src/shared/auth/jwt-payload.interface'; import { OptionalJwtAuthGuard } from 'src/shared/auth/optional.guard'; import { RealIP } from 'src/shared/auth/real-ip.decorator'; -import { hasRoleAccess, RoleGuard } from 'src/shared/auth/role.guard'; +import { hasStaffAccess, RoleGuard } from 'src/shared/auth/role.guard'; import { isUserActive, UserActiveGuard } from 'src/shared/auth/user-active.guard'; import { UserRole } from 'src/shared/auth/user-role.enum'; import { CLIENT_HEADER } from 'src/shared/utils/request-client'; @@ -205,7 +205,9 @@ export class SupportIssueController { // Staff routing requires an active account: blocked staff keep their JWT role until token // expiry (default 2d) but must not be able to post official replies. Non-staff callers // (including anonymous, since the guard is Optional) fall through to createMessage. - if (jwt?.role && hasRoleAccess(UserRole.SUPPORT, jwt.role) && isUserActive(jwt)) { + // `hasStaffAccess`, not `hasRoleAccess`: this route is OptionalJwtAuthGuard-only, so the staff KYC gate + // has to be applied here — posting official DFX replies is a staff privilege like any other. + if (jwt?.role && hasStaffAccess(UserRole.SUPPORT, jwt) && isUserActive(jwt)) { // Mail-origin staff sessions must complete STRICT 2FA before posting an official reply. The global // TfaEnforcementInterceptor already enforces this invariant on every route; this inline check is kept as // defense-in-depth on this sensitive sink. Wallet-signature logins (no tfaRequired) are unaffected. From bc984ebb0d50f54a33024cfb9b657fe42dce796d Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Sat, 1 Aug 2026 10:56:16 +0200 Subject: [PATCH 2/4] 81e8af98 - fix(liquidity-management): resolve uncertain Scrypt orders without a person (#4440) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(liquidity-management): bound the uncertain-order quarantine An order the venue keeps having no record of stayed UNCERTAIN indefinitely: the only way out was a manual release through the admin endpoint. Where nobody performs that release, the order never resolves and the rule behind it stays blocked, so the venue silently stops being served at all. Abandon such an order to FAILED once it has been unresolvable for longer than ABANDON_UNRESOLVED_MINUTES. What makes that safe is not a conclusion about the order but that a rule replans from the venue's CURRENT balance, never from the abandoned order: an execution that did happen has already moved that balance, so the replan sizes itself down rather than duplicating the request. Two guards, both fail-closed: - UNAVAILABLE never abandons, however old the order. It is the absence of an answer, not an answer, and waiting cannot turn it into one. - An order without a creation date never abandons. Util.minutesDiff reads a missing date as the epoch and would otherwise abandon instantly. * fix(liquidity-management): scale the abandon bound to the request kind One bound for every command was either useless or unsafe. Measured over the 30 days to 2026-07-29 on completed Scrypt orders in prod: trades (n=55): median 9.6s p95 19.8s max 57.1s withdrawals (n=49): median 7.7min p95 82min max 5.6h A trade is done inside a minute, so holding one for hours strands its rule for no reason — the incident order was a trade and sat for nine. A withdrawal at 30 minutes is entirely ordinary, so a short bound there would abandon live transfers and reissue them. Trades now abandon after 5 minutes (~5x the slowest observed), everything else after 12 hours (~2x the slowest observed withdrawal). Balances refresh every minute and the pipeline runs every 10 seconds, so neither bound is limited by detection speed, only by how long the request may still be alive. Venue-internal commands are an allowlist (buy, sell, purchase): an unrecognised or unloaded command falls to the long bound, because being slow to abandon costs minutes while being fast to abandon a live transfer duplicates it. * fix(liquidity-management): bound orders no integration can look up The bound was applied only after a lookup had actually been made, so it missed the one case where waiting is provably pointless: an order whose adapter is no longer registered. No lookup can ever arrive for it, and without a manual release nothing in this system moves it again — the same permanent quarantine, reached sooner. Apply the same bound there. Also corrects a comment that said "for hours", which stopped being true once trades got a five-minute bound. * fix(liquidity-management): correct the abandon bound after review Three defects, all in the abandon path introduced by this PR. Clock ran from the wrong instant. `unresolvableTooLong` measured from `created`, a @CreateDateColumn that is never updated. An order can run normally for a long time and only become UNCERTAIN late, when a completion check amends or restarts it and that write goes unconfirmed — its `created` is old by then, so the bound expired on the very first pass while the replacement request was seconds old and plausibly still arriving. Measures from `updated` now, which moves with the transition into quarantine. Allowlist keyed on the command name alone. `sell` on an exchange is a book match settled in seconds; `sell` on the DEX adapter is an on-chain swap with a confirmation time, and it was getting the five-minute bound. Keyed on `system/command` pairs now, listing only the exchange trades actually measured. Weakest write had the weakest guard. `abandonUnresolvableOrder` called `leaveQuarantine` without `expectedRecheckDue`, so an operator release written between this pass's read and the write was overwritten together with its audited reason — the one write resting on no evidence outranking the one resting on a person. Narrowed on the examined release, as `completeNotSentRelease` already does. Also withdraws the automatic abandon for orders no integration can look up. That path never asks a venue, so only the clock would be left, and the safety argument requires a balance that reflects an execution: a chain balance omits unconfirmed transactions and a bank balance comes from the last imported batch. Abandoning there would be guessing with the one class of order nothing can observe, so it keeps waiting for the manual release instead. * fix(liquidity-management): match SQL NULL when narrowing on an absent release The narrowed update never matched a row. TypeORM renders a raw null in a where object as `= NULL` — invalidWhereValuesBehavior.null defaults to "ignore", which falls through to an equality — and `x = NULL` is UNKNOWN in SQL, so it matches nothing, not even the row whose column really is NULL. That disabled this PR's entire purpose. The abandon path is only ever reached for an order nobody had released, so the examined value there is always null: every abandon would have updated zero rows and logged itself as a lost race while the order stayed quarantined. `completeNotSentRelease` carried the same defect on its NOT_SENT branch, which reaches it with no release pending. Uses IsNull() for the null case, matching how the rest of the repo expresses this. The mocked repo in the unit tests returns a fixed affected count and cannot see criteria semantics, so the test asserts on the operator itself. Also corrects the UNCERTAIN status comment, which claimed quarantine is always time-bounded. Since the previous commit withdrew the automatic abandon for orders no integration can look up, two cases legitimately wait for a human: those orders, and a venue that stays unreachable. * docs(liquidity-management): align the remaining comments with the new bound Three places still described the pre-change behaviour, one of them contradicting a comment corrected earlier in the same method. - resolveUncertainOrders said anything inconclusive "stays put"; it now stays put only until the bound for its kind of request runs out. - The UNRESOLVED enum value said "look again later", implying indefinitely. - ScryptAdapter.resolveUncertainOrder's docstring said a missing record leaves the order quarantined for a human, while its own inline comment already said the caller bounds that wait. Two weaker statements in the same adapter are softened for the same reason: they describe entry into quarantine, which is unchanged, but read as if quarantine were the end state. Also adds the missing regression test for the NOT_SENT caller of the IsNull narrowing. Asserting on status alone cannot catch a regression there, because resolveAsNotSent sets FAILED synchronously before the write is attempted, and the mocked repository returns a fixed affected count. * docs(liquidity-management): finish aligning comments, cover both allowlist entries Remaining statements that the abandon bound made untrue, plus two test gaps. Comments: the abandon docstring said "for hours", which is wrong for the five-minute trade bound; a stray line break left "The" stranded mid-sentence; the manual-release docstring claimed an unsent request would block its rule forever, which now only holds for the cases nothing can observe; three adapter comments described quarantine as ending only with a human. The resolveUncertainOrder docstring also said it can only ever confirm a positive, which its own NOT_SENT branch contradicts — that one predates this PR but sits in the paragraph already being corrected. Tests: the bound tests only ever used `sell`, so dropping or mistyping `scrypt/buy` from the allowlist would have gone unnoticed; they now run over both entries. And nothing covered a pending release together with an expired bound — the branch order makes the audited release win, but only a test stops a later reordering from silently replacing an operator's recorded verdict with an anonymous clock. * fix(liquidity-management): report an unaskable order as unavailable, not unresolved An order with no correlation id was reported as UNRESOLVED, which means the venue answered and had no record. Nothing was asked: there is no reference to ask about. The distinction now decides an order's fate — the caller may abandon an UNRESOLVED order once its bound expires, and would have recorded "the venue has had no record of it" for a lookup that never ran, against this PR's own rule that a row must not claim an observation nobody made. UNAVAILABLE is exactly this case by its own definition: the venue could not be asked at all, and such an order is never abandoned automatically. Only orders predating the reserve-before-send guarantee can reach it, and those are the ones that most need a person to look. Also fixes a docstring this PR made worse two commits ago: "an absence can only ever confirm a positive" states the opposite of what the code does. An absence confirms nothing — that is the whole point of the paragraph. * test(liquidity-management): expect an unaskable order to be unavailable The existing case asserted UNRESOLVED for an order with no reference, without saying why. It was written when the distinction had no consequence: nothing acted on UNRESOLVED, so either value behaved the same. This PR gives it one — UNRESOLVED now permits an automatic abandon — so the old expectation locks in the behaviour the previous commit corrected. Replaced by the documented case, covering both an undefined and a null reference, and asserting that no lookup is attempted at all. * fix(liquidity-management): do not abandon an order with references left unasked The lookup stops at the newest reference when the venue cannot see it, and deliberately does not fall through to the one it replaced — a hit there would report SENT on a superseded reference. But it reported that stop as UNRESOLVED, an answer, and the bound may abandon an order on an answer. The predecessor of an invisible replacement is very often the live one, so an order with an open attempt at the venue could have been failed after five minutes and its rule sent to plan again. An existing test documents exactly this shape: predecessor still NEW at the venue. It now reports UNAVAILABLE whenever references were left unasked, which the caller never abandons on. A single unknown reference still reports UNRESOLVED — nothing was skipped there, so the answer is complete. Also repairs a side effect of the previous commit: with an unaskable order reporting UNAVAILABLE, a hand-verified release stopped taking effect immediately and instead waited out the full unreachable-venue hour. Nothing went out under a reference, so no lookup can ever answer — the same dead end as an order with no integration, and released the same way. Two smaller review points: the it.each table left its parameter implicitly any, and the allowlist hardcoded the system name that already exists as an enum value. The command halves stay literal because their enums live in the adapters, which import this entity. * fix(liquidity-management): count only references that can still be live as unasked The previous commit counted every remaining reference as unasked, including ones already superseded. That creates the permanent quarantine this PR exists to end: once a replacement has been adopted, the reference it replaced stays in the list forever, so if the venue later loses sight of the adopted one, the dead predecessor makes every lookup incomplete and the order can never be abandoned — worse than before this PR, where the bound would have applied. Only references at least as current as the adopted one count now, matching how adoptLiveReplacement already separates claimed replacements from established ones. Also completes two comments: UNAVAILABLE has three causes, not one, and only two of them were named. A complete answer may time out; an incomplete one never does, whatever caused it. * feat(liquidity-management): bound the unanswered outcomes too Waiting for an operator was still the end state whenever the venue could not be asked, or not completely: no reference to ask with, an unreachable venue, a lookup that stopped with a reference left unasked. Where nobody performs the manual release, that is not caution — it is the same permanent quarantine this PR set out to remove, reached by a different door, and the rule behind the order dies with it just as surely. These outcomes now expire as well, on their own much longer clock: 24 hours, over four times the slowest order ever observed at this venue. By then nothing can still be in flight, so the balance the rule replans from already reflects whatever really happened — the same argument that makes the short bound safe, with far more room. The recorded reason says the venue was never heard from, never that nothing was sent, so the row still claims only what was observed. One case still waits for a person, and deliberately: an order no integration can look up at all. There the venue is never asked and the balance a replan would read is not live either, so neither half of the safety argument holds. * fix(liquidity-management): finish a lookup that would otherwise never complete The 24-hour bound did not hold for one of the three cases it covered. When a claimed replacement never reached the venue, it is absent on every pass — so the lookup stopped at it every time and never reached the reference it replaced. Waiting could not fix that: the absence is structural, not transient. Meanwhile these are GTC limit orders with no expiry, so the unchecked predecessor can sit open in the book indefinitely, and abandoning after 24 hours would have planned those funds a second time. The measurements behind the bounds cannot see this either. They describe orders that completed, so an order that never becomes observable is by construction absent from the sample — the numbers are a floor, not a ceiling. Past the age at which this venue already treats an order as lost, the invisible replacement is now recorded as spent and the lookup carries on to the reference that may actually be live. That turns a question which never completed into one that does, which is the only way such an order ever gets a real answer rather than a guess. Review points from the same round: the two abandon paths were identical except for their reason, so they are one method taking it as an argument; the timeout map and the entity mutator no longer name only the case they started with; a JSDoc had come to sit above the wrong method; and the new predicates now have direct entity tests covering both bounds and the missing-timestamp guard. * fix(liquidity-management): apply the same age rule to the write path The previous commit repaired the reconciliation lookup but not the completion check, and the two disagreed. Reconciliation would walk past a claimed replacement the venue never showed and resolve the order to its predecessor; the very next completion check blocked on that same invisible claim, refused every write, and quarantined the order again. The order oscillated between the two states indefinitely — and because each return to quarantine refreshes `updated`, both abandon bounds were reset every cycle, so the safety net meant to end exactly this was disabled by the loop itself. adoptLiveReplacement now steps past a claim under the same condition the lookup uses: the venue ANSWERED and still does not show it, past the age at which this integration already treats an order as lost. A claim the venue could not be asked about keeps blocking however old it is — silence is not an answer, and no clock turns it into one. Also drops a call to recordSpentCorrelationId that could never do anything: its argument always came from the order's own reference list, which is the one case the entity guards against. The commit message claiming otherwise was wrong; the correctness came from continuing the loop, not from that line. The "keeping it quarantined" warning now only logs on the path that actually keeps it quarantined, and the comment claiming an explicit rejection is the only way to reach an older reference names the timeout path too. * feat(liquidity-management): cancel before giving up, instead of timing it out Everything about abandoning a quarantined order was an estimate of when its request could no longer execute. That estimate is what three rounds of review kept breaking: the measurements behind it only describe orders that finished, Scrypt trades are GTC and expire never, and the clock the two code paths shared was a whole-row timestamp that any unrelated write reset — so the order oscillated instead of ending. None of that is necessary. What makes abandoning dangerous is not the passage of time but the possibility of a live request, and that possibility can be removed rather than waited out: cancel every reference the order ever sent, and only give up once the venue confirms none of them can execute. A cancel is the opposite of a re-send and cannot create anything, which makes it the one write that is always safe against an outcome nobody observed. The order is abandoned when the venue settles all of its references, and keeps waiting when it will not — unreachable, or reporting one in some other state. That is the same conservative answer as before, but reached for a stated reason instead of an assumed one, and it no longer depends on any clock. What this removes: the 24-hour unanswered bound, the reference-left-unasked classification, the age exception in adoptLiveReplacement, and the tests built on them. The two measured bounds stay — they still decide when it is worth trying to clean up at all, not whether cleaning up is safe. Scrypt only. Adapters that cannot cancel omit the new hook and their orders keep waiting for a person, unchanged. * fix(liquidity-management): read what a cancellation actually established The cancellation had two outcomes where the venue has three, and each gap was a way to lose money or to stall forever. A partially filled order cancels with a terminal Canceled status AND a non-zero fill — the protocol says so explicitly. Reading only the status called that "nothing can execute" and abandoned the order, dropping a real fill and handing the rule back funds that were already spent. A fill is now recognised whatever the final status says, and the order is pointed at the reference that executed so the ordinary completion path books it instead. A refused cancellation arrives as an execution report, not as an error — this venue sends no separate reject message. So the branch meant to catch "there is no such order" was unreachable, and the incident case this PR exists for would never have been settled at all. It is now read off ExecType and CxlRejReason, and only UnknownOrder counts: every other reason settles nothing. An order with no reference ran the loop zero times and fell through to "all settled" — abandoned on a confirmation nobody gave. It now refuses without asking anything, since there is nothing to ask about. Also adds the tests this contract had none of, and corrects the comments left describing the two-tier bound that no longer exists: age only decides when cleaning up is worth attempting, and the venue decides whether giving up is safe. * test(liquidity-management): fix fixtures for the new cancellation contract The cancel now returns the venue's execution report rather than a boolean, so a mock that resolved undefined made the amend path read a field off nothing and take the unconfirmed branch. And cancelOutstanding derives the trade pair from the rule, which the shared uncertain-order fixture does not carry. * fix(liquidity-management): file a cancellation under the order it settles The fill was still unbookable, one layer further down. This venue tags a cancel confirmation with the CANCEL request's id and names the cancelled order only in OrigClOrdID — while every lookup here is keyed on ClOrdID alone. So the terminal state was filed under an id nothing ever asks about, and the cancelled order kept answering with whatever non-terminal report preceded it. Pointing the row at a reference that had executed therefore led nowhere: reconciliation saw the stale report, handed the order back, and the completion check found the same stale report again. The confirmation is now filed under the order it settles. Even then the fill was unreachable: the reconciliation lookup stops at the first reference the venue does not show, newest first, and a claimed replacement that never arrived is exactly such a reference. It would stop there on every pass and never reach the older one the cancellation had just identified as executed. The stop now applies only to references NEWER than the one the row names — the current reference is authoritative, and a cancellation that found a fill points at it deliberately. An unreadable filled size no longer counts as zero; it settles nothing, since concluding "nothing filled" from a value that could not be parsed is how a real fill gets dropped. The rejection branch in the catch is gone — this venue answers a refused cancel with a report, not an exception, so it was unreachable and contradicted the branch that does the work. Adds seven direct tests for the classification itself, which until now was only ever mocked, and completes the fixture that the type checker could not see through. * fix(exchange): file only a real cancellation under the cancelled reference The previous commit filed every response tagged with the cancelled order's id, refusals included. A refusal comes back on the same channel carrying the order's last known state, and for a reference the venue never had that reads as a live New order — so caching it invented one. The next lookup found that phantom, called the order sent against a reference that had never executed, and pointed the row away from the one that actually filled. Same lost fill as before, one layer further down. Only a terminal Canceled says anything about the order being cancelled, and that covers both outcomes the caller distinguishes: nothing filled, and partially filled. Refusals stay uncached, so a reference the venue never had keeps reporting nothing — which is what lets the newer-reference guard in the reconciliation lookup do its job. Adds the two tests that would have caught it: a refusal must leave the cache untouched, and a cancellation that finds a fill must still name the executed reference after the next reconciliation pass. * feat(liquidity-management): treat an executed reference as settled too The whole tangle came from one wrong assumption: that a reference which filled is a reason to hold on to the order. It is not. The question this path asks is whether a reference can still execute, and one that already ran answers it just as finally as one that was called off. Holding on was also pointless, because the fill is in the venue's balance and that balance is what the rule replans from — it plans for what is actually left, not for what this row believed. So there is nothing to rescue and nothing to guess: cancel everything, give up, replan. That removes the part which produced a critical finding in five consecutive review rounds — rerouting the row onto the executed reference, and with it the need to remember which references the venue never had. Gone with it: the reroute, the newer-than-current guard in the lookup, and the chained test that only ever existed to cover them. Two findings from the same round, both real: - The cancel waiter accepted the first report mentioning the order, including a non-terminal PendingCancel. Since it unsubscribes on its first match, the real terminal report that follows would never have been seen. PendingCancel is now skipped. - resolveUncertainOrders described itself as only ever observing, which stopped being true when this PR gave it a cancel. * fix(exchange): require a terminal state before calling a reference executed The fill was read before the status, so a REFUSED cancel could be classified as executed. A refusal carries the order's last known state — a partially filled order that could not be cancelled reports a fill while staying wide open — and since the caller now treats executed like settled, that would have let it walk away from a reference still able to trade. The same double execution this path exists to prevent, re-entered through the door the previous simplification opened. A fill on its own says nothing; only a terminal state says nothing more is coming. Executed now requires Canceled or Filled, and a non-terminal report with a fill settles nothing. Also records which reference filled on the order before abandoning it. The money side is carried independently by the venue's own transaction sync, but an abandoned order books no output and cannot say for itself what happened — naming the reference is what lets the two be tied together afterwards. Documents the one assumption that stays an assumption: the replan reads a pushed balance, so a just-landed fill may briefly not be in it. Seconds in practice, but a window rather than a guarantee, and the comment said otherwise. Corrects the enum, the docstring and the log line that still described the executed case as one that must be completed rather than abandoned. * fix(exchange): decide what counts as terminal in one place The cancellation check restated which states are final and left one out. A rejected order is as unable to trade as a cancelled one, but it fell through both branches and settled nothing — so an order that provably could not execute would have stayed quarantined for want of being recognised. Exactly the class of permanent hang this PR removes, reintroduced beside it. This venue already answers that question in one place, and that answer includes rejected. Reusing it means a second list cannot drift from the first. Adds the two cases that were missing: a rejected reference with nothing filled settles, one with a fill counts as executed — same as cancelled and filled. * docs(liquidity-management): align three statements with what the code now does All three were true when written and were made false by changes in this PR. The entity header still claimed abandoning is safe because the replan reads the CURRENT balance — stated as settled fact, while the adapter's own cancellation now honestly calls that a window rather than a guarantee. It names both things that actually carry it now: the venue's confirmation, and a replan against the balance, with the freshness caveat pointed at where it is described. The no-reference comment said such orders resolve on a bound rather than a verdict. They do not: with no reference there is nothing to cancel either, so the automatic route cannot confirm anything about them and they wait for a person. And abandoning is no longer reserved for orders the venue never accounted for — it is reached just as well when a reference executed and nothing is left outstanding. * docs(exchange): state the UnknownOrder reading as the inference it is The comment asserted that UnknownOrder means there is no such order, so nothing under that reference can execute. The protocol spec lists the reason without defining it, so that cannot be read off the value: "never existed" and "not processed yet" are indistinguishable from it alone. The reading itself stays — it is what lets the incident case resolve at all, and every alternative leaves the order waiting forever. But it now says what it rests on instead of claiming certainty: the caller only cancels a reference it has already failed to find through a separate status lookup, and only after the order has outlived the window in which its request could still be live. Two independent negative answers plus that age are the strongest evidence this protocol offers, which is a different claim from proof. * fix(exchange): treat a missing filled size as missing, not as zero Number('') is 0, not NaN, so an empty or whitespace CumQty passed the finite check and read as an untouched order — settling it. The comment two lines above promises the opposite: that an unreadable quantity settles nothing. Emptiness is now caught separately, with tests for both forms. Also corrects five statements that this PR had made untrue: The entity's abandon mutator still said "an order the venue never accounted for" — the fourth instance of a sentence corrected three times elsewhere, and wrong for the same reason: abandoning now also happens when a reference executed. The abandon docstring claimed to rest on nothing but the clock, four lines above stating correctly that it is reached only after the venue confirmed nothing can execute. The clock decides when cleaning up is worth attempting; the confirmation is what permits it. Same wording corrected in the sibling docstring and in the test comment that echoed it. The UNRESOLVED enum value said the bound alone gives an order up. It does not — without the cancellation's confirmation the order stays quarantined. And "every reference the row ever put on the wire" overstates the source: a reference is reserved before it is sent, so the list can contain one that never left. * fix(exchange): a refusal that contradicts itself settles nothing The UnknownOrder branch returned settled unconditionally, while the terminal branch right above it checks the filled quantity before deciding. That inconsistency mattered: a refused cancel carries the order's last known state, and the neighbouring test proves such a refusal really can arrive with a filled quantity attached. A report claiming the venue has no record of an order while reporting a fill on it disagrees with itself. Nothing may be concluded from that — least of all that nothing can execute, which is the one conclusion that lets the caller walk away. It now settles nothing and the order keeps waiting. * docs: say what the cancellation evidence actually covers The UnknownOrder comment claimed two independent negative answers per reference. It gets one: the status lookup stops at the first reference the venue does not show, while the cancellation asks about every reference of the order — so for all but one of them the refusal is the only negative answer there is. Age plus one refusal is still the strongest evidence this protocol offers, but that is a weaker claim than the one that stood there. Four statements corrected in the same class, all of them versions of sentences already fixed elsewhere in this PR and missed at these spots: - The reference-list helper still said "actually put on the wire", one line from the caller where exactly that wording was corrected — a reference is reserved before it is sent. - The interface contract for cancelOutstanding had the same wording, making it narrower than what its only implementation documents. - A comment still had the bound abandoning an order by itself; it triggers a cancellation attempt, and only that attempt's confirmation abandons. - An inline comment still called the abandon write "resting on no evidence at all", four lines below a docstring that had already been corrected to say it rests on the venue's confirmation. * fix(exchange): check the contradiction before the status decides The self-contradiction guard added last commit sat behind the terminal-state check, so it only caught the non-terminal shape. A refusal claiming the venue has no record while reporting a fill is contradictory whatever status rides along — with a terminal one attached it went straight through as executed, which is exactly the permission to give the order up that the guard exists to withhold. It now runs first. The cancellation is also no longer cached when its filled quantity is unreadable. Readers derive that value with `parseFloat(...) || 0`, so such an entry would quietly claim nothing was filled on every later lookup — the same reading the classification itself was changed to avoid. And three statements the previous commit claimed to have corrected but did not: its edit script aborted partway and I only chased the one failure it reported instead of checking whether the rest had run. The interface contract still said "put on the wire", and two comments still had the bound giving an order up by itself rather than triggering a cancellation whose confirmation does. Corrected now, with a grep over the whole diff to confirm no further instances remain. * test(exchange): cover the matcher and a non-numeric filled size The PendingCancel skip added earlier had no test at all: every cancellation test mocks the transport directly and hands back a finished report, which routes around the matcher entirely. This one captures the matcher and drives it, asserting it ignores the interim state and takes the terminal report that follows — the waiter resolves on its first match and stops listening, so accepting PendingCancel would freeze it as the answer. Also adds a genuinely non-numeric filled size. Empty and whitespace were covered, 'abc' was not. * fix(liquidity-management): ask every reference before deciding cancelOutstanding promised to cancel everything the order could still have live, then left the loop at the first reference the venue would not settle. Those it skipped are precisely the ones that can sit open in the book while the newest keeps refusing — so the method did the opposite of what it says whenever it mattered most. It now asks about every reference and decides afterwards; one refusal still keeps the whole order quarantined. The existing test hid this: its unconfirmed reference happened to be last in iteration order, so both got called anyway. The new one puts it first. Also sharpens the interface contract on what "confirmed" covers. An accepted cancellation or a terminal order settles the question outright; a refusal saying the venue has no such order is an inference from its own words, not a statement about execution. The contract said CONFIRMED for all three, which overstates the weakest of them. * docs: correct the bound's role everywhere, and say where it does not reach The previous commit claimed a grep over the whole diff had confirmed no further instances of "the bound gives the order up". It had not — the grep matched the exact wording I had just changed, not the pattern. Five more places said it, in three different phrasings. Searched by pattern this time, with the result verified as empty afterwards. More useful than the wording: the transfer bound does not reach Scrypt withdrawals at all. It was derived from measured withdrawal runtimes, but reaching it only triggers a cancellation attempt, and the one venue that implements cancellation refuses it for withdrawals outright — there is no such thing as cancelling one there. So those still wait for a person, and the value governs the other transfer kinds and any venue that gains a cancellable withdrawal later. Said so at the constant, and renamed the test that asserted this to stop implying a venue can do what Scrypt cannot. Also: the abandon write does rest on evidence — the venue's cancellation — just not on evidence about whether the request was ever sent. And the cache test now covers all four unreadable shapes instead of only the empty string. * fix(liquidity-management): stop calling an inference a confirmation The reason written into the order and the log said the venue "confirmed" that nothing can execute. It does not always confirm that: a refusal saying it has no such order settles the question by inference, which the code says plainly where that inference is made. The interface contract was corrected for exactly this distinction two commits ago and the downstream message was not, so an operator reading the reason later would lose the difference between an accepted cancellation and something concluded from the venue's wording. Both now say "answered", which covers all three outcomes honestly. Four smaller consistency fixes in the same class: two docstrings, an enum comment and a test title said "every reference the order ever sent", excluding the reserved-but-never-sent case that cancelOutstanding explicitly covers; a trade test title still credited the bound alone with abandoning, while its transfer twin had been corrected for that; and a test comment still said the caller may abandon once the bound expires, without the settled cancellation that actually permits it. * fix(exchange): never file a cleanup cancellation under the order it cancelled A confirmed cancellation was cached under the cancelled order's own id. That made a later status lookup read the order as terminally cancelled — and a cleanup cancellation says nothing about the order as a whole: its sibling references may still be unsettled and live. The lookup would then report the order as known, take it out of quarantine, and the completion check would open a replacement beside a reference that can still trade. The double execution this whole path exists to prevent, reachable in a plain multi-reference order. The aliasing only ever existed to make the EXECUTED reroute work, and that reroute is gone — cancelling now settles a reference rather than adopting it. So this is dead code with a live hazard, and removing it restores what the method did before this PR: report the outcome, cache nothing. The test that demanded the aliasing now demands its absence. * test(exchange): restore the amend-propagation guard lost to a regex My delete regex in the previous commit took more than the one test it was aimed at. I restored the block brace and the helper it had swallowed, but not this test — it guards that an unconfirmed amend is propagated as such and that nothing is cancelled along the way, behaviour that is unchanged and still thrown in two places. Its loss also orphaned an import, which eslint reports as a warning; `npm run lint` exits zero on warnings, so my own gate check called that green while CONTRIBUTING requires none. Restored verbatim from the parent commit. Verified by diffing every test title against that commit: only the one intentional reversal differs, and eslint with --max-warnings 0 is clean. * style(exchange): keep these two files on the CRLF endings this repository uses Restores what my Python edit scripts had silently normalised to LF. It is not cosmetic: with every line counted as changed the real diff was unreadable and the branch could not take a thirteen-line change to the same region. * fix(liquidity-management): keep the venue-lookup cooldown inside the abandon bound #4438 landed on develop while this branch was open and throttles the very loop this branch extends: a quarantined order's venue lookup now waits a tenth of the order's age, capped at thirty minutes, and skips the pass entirely while that wait runs. The skip happens before the branch that gives an expired order up, so the throttle silently governed the deadline too — a trade quarantined when it was already hours old would draw the full thirty-minute interval and be abandoned six times its own five-minute bound too late, with nothing anywhere saying the bound had moved. The interval is now also capped at what is left of the order's own bound, held at the cooldown floor once that has passed. A wait reaching past the deadline it has to keep cannot do its job; below the floor, a cancellation the venue will not confirm would retry on every ten-second tick, which is what #4438 exists to prevent. Both bounds keep their intent and the tighter one wins. The remaining time comes from the entity, from the same private accessor as the deadline itself, so the two can never disagree about which bound applies. Only the direction carrying the safety is asserted: an order past its bound reports nothing left. The converse cannot hold at the single instant where elapsed equals the bound, and a non-negative duration cannot express that — it costs one cooldown floor, never a missed abandonment. Verified by mutation: removing the new cap leaves the abandonment sitting for the full thirty minutes and turns the test red. #4438's own cap tests are untouched — their fixture sets no `updated`, so no deadline constrains them — and a transfer, whose bound is twelve hours, still waits the full cap. * fix(exchange): refuse a negative filled size instead of reading it as untouched A cumulative filled size below zero is finite and parses cleanly, so it passed the unreadable-value guard and then lost to `filled > 0` — reported as an order nothing had ever traded, which settles the cancellation and lets the caller walk away from the reference. Same class as the `Number('') === 0` case the guard above it already names: a value that cannot mean what it says must not be compared as though it could. Fails closed as UNCONFIRMED. Verified by mutation: dropping the check turns the two new cases green again, which is what makes them worth having. * test(liquidity-management): type the new cooldown fixtures through the shared helper `tsc` rejected what jest had happily transpiled: an inline action literal is checked against LiquidityManagementAction inside Partial and lacks paramMap, created and updated. The file already has agedOrder(minutes, command) for exactly this, with the cast the rest of the suite uses, so both new tests now go through it instead of building their own fixture. My local gate had missed it because I ran jest and eslint but not tsc. Re-verified by mutation after the rewrite: removing the cap still turns the bound test red. * fix(liquidity-management): freeze the deadline cap at the last lookup The cap was read fresh on every tick, so it shrank while the wait it bounds grew — and the two met halfway. A five-minute bound was therefore re-asked at 2.5 minutes, then 3.75, then 4.4: a geometric series of full history fetches before a deadline that had not moved, which is the cost #4438 exists to avoid. The abandonment also landed late, around 5.75 minutes, because each of those lookups restamped the cooldown. Now measured as the time that was left when the last lookup finished. Adding the elapsed wait back to what is left now yields exactly that figure — bound − (attempt − updated) — so the cap holds still between lookups and needs nothing stored beyond the stamp already kept. One lookup, and the abandonment lands on the bound rather than past it. Verified by mutation, both directions: re-reading the cap on each tick turns the new test red, and removing the cap altogether turns two red. * fix(liquidity-management): stop reporting a stuck quarantine write as a race A missed compare-and-set was logged as "already resolved elsewhere, skipping", which is one of two possible causes and the harmless one. The other is a narrowing that matched no row and never will: then the same line appears on every pass while the order does not move at all, and the wording makes a permanent block read like routine concurrency. Found the hard way in production. A release timestamp written by hand in SQL carried microsecond precision; a JavaScript Date carries milliseconds, so the value the code loaded could not match the row it came from. The order sat behind this message, once per tick, until the precision was corrected — and the log said it had been resolved elsewhere the whole time. The line now names both causes, says that repetition means the second one, and is a warning: a race is rare and self-correcting, while the other case is a silent permanent block and this is the only trace it leaves. Which is the exact failure mode this branch exists to end, so it should not be introduced by the branch's own bookkeeping. No extra query — re-reading the row to tell the cases apart consumed a call the surrounding tests sequence their mocks on, which is too much machinery for a log line. Verified by mutation: restoring the old wording turns the new test red. * fix(liquidity-management): give the order up at its bound, not a cooldown later Two review lanes found the same defect independently, and between them the whole of it: the quarantine bound was reliably overshot by up to a minute. Half of it was an off-by-one between the two halves of one clock. The deadline having arrived is what lets a reconciliation pass run at all; being *past* the bound is what lets that pass give the order up. With a strict `>` the instant the bound was exactly met satisfied the first and not the second, so the pass ran, abandoned nothing, and re-stamped its cooldown — a five-minute bound gave up at six. Now `>=`, and the two turn over together. Asserted with the clock held still, because anything built from a live `Date.now()` is already a fraction past the boundary, where `>` and `>=` agree and the bug is invisible. The other half was the cap's own floor. A lookup running shortly before the deadline has less than a floor's worth of time left, and imposing a full interval on it scheduled the next pass after the deadline — the overshoot the cap exists to prevent, caused by the cap. The floor now applies only once the deadline has passed, where there is nothing left to protect and an unconfirmable cancellation must still not retry every tick. Making that distinction possible is why `msUntilAbandonable()` became `abandonableAt()`. A remaining duration has to be clamped at zero, and that clamp erases precisely what a throttle needs after the deadline: whether the wait it is about to impose began before it. An absolute instant carries the sign, so the lookup's headroom is a subtraction rather than a reconstruction — and the comment explaining the reconstruction is gone with it. Verified by mutation, three ways: restoring the floor on an open deadline turns the new boundary test red, `>` turns three red, and dropping the cap turns three red. My first attempt at the floor half was not caught by its own test — the scenario never produced a lookup close to the deadline — so the test now reaches it the way production does, via an order that enters the pass already near its bound with no cooldown recorded. * refactor(liquidity-management): follow the method naming rule, fix a stale doc link CONTRIBUTING requires methods to be verbs, with the documented exception being boolean getters on `is`/`has`. `abandonableAt()` is neither, and it returns a Date — so `getAbandonableAt()`. The entity's other additions here are already verbs or read as statements; this was the one value getter, and the only method of its shape in any entity, so there was no house pattern arguing the other way. The JSDoc above `unresolvableTooLong()` still linked `{@link msUntilAbandonable}`, a method the previous commit removed. A dead link in the one comment explaining why the two halves of that clock must agree is worse than no link. * refactor(liquidity-management): apply the naming rule to the helper too `abandonBoundMinutes()` is a noun phrase, exactly what the previous commit renamed `abandonableAt()` for. Applying the rule to the public getter and not to the private helper beside it would have left the file arguing with itself. The entity spec's describe block also still carried the old name. The local variable holding the returned instant keeps its noun name: the rule governs methods, and `abandonableAt` is what that value is. * docs: stop promising a bound the code deliberately does not give Three statements claimed more than the code does. The bound was described as "how long before it is abandoned", the quarantine as ending "not indefinitely", and every outcome at an askable venue as "bounded". None of that holds: reaching the bound starts a cancellation, and the order is given up only if the venue settles every reference it claimed. Where an integration cannot cancel, or the venue will not answer, the order stays quarantined past the bound — which the same comments then said correctly a few lines further down, so they contradicted themselves. The claim these bounds actually support is narrower and worth stating exactly: they end the assumption that somebody will eventually look, not the quarantine. The catch in `cancelOrder` said anything reaching it is a transport failure. It starts with a trade-pair lookup, so a missing pair or configuration lands there too, having sent nothing. The comment now says what the handler really knows — that it cannot tell which failures got as far as writing, which is precisely why it drops the cached report for all of them. Comments only; no statement, condition or value changed. * docs: finish the pass, and check the whole class rather than the reports Three more statements of the same kind the previous commit corrected, each contradicted by a comment within a few lines of it: - The UNCERTAIN enum said an order with no venue record "is abandoned to FAILED" once past the window, four lines above its own correct note that only the venue's answer permits FAILED. - `completeNotSentRelease` said the other exit means an unreleased order "cannot block its rule forever", against the corrected text 150 lines up in the same file saying it can, and that this release is then the only way out. - SETTLED was documented as "the certainty the caller needs" in both the enum and the service, while the constant it rests on says the unknown-order reading is an inference and not a documented guarantee. Both now separate the two qualities of answer instead of flattening them. Fixing only what was reported is what left these behind twice, so this time the whole diff was swept for the pattern — every new comment claiming a guarantee. The three above were real. The remaining matches are sound: one is asserted by the test it appears in, one is hedged with "in practice" and exists precisely to explain why the code does not rely on it, and the rest describe the failure this branch fixes rather than promising anything about it. Comments only; no statement, condition or value changed. * docs: correct three claims, including one where the review was right and I was not The review's first two rounds on this pattern each left something behind, and one of the leftovers was a call I had made myself: - The UNCERTAIN enum said an order past the window "has its references cancelled". Still too much: a Scrypt withdrawal returns false from cancelOutstanding without asking anything, and an adapter that cannot cancel omits the method entirely. It now says an attempt is made where the adapter supports one. - The entity said an askable venue leaves no outcome waiting for an operator. The attempt is a cancellation; a venue that will not confirm one holds the order indefinitely, and a request that cannot be cancelled is never attempted. Stated as what it is: those orders get an attempt, not a bound. - `abandonUncertainOrder` claimed its release marker is "always null in practice, because a pending release routes to an earlier branch". I had reviewed that claim in the previous commit's sweep and passed it. It is wrong: this branch has no release condition of its own, so an order released while the venue was unreachable reaches it with the marker set. The compare-and-set was already correct — it narrows on the value actually read — but the comment explained it with a premise that does not hold. Checked what the third one implied and it is fine: the abandonment prefixes the existing message rather than replacing it, so an operator's audited reason and ticket survive. Said so in the comment, since the question now arises there. Comments only; no statement, condition or value changed. * docs(liquidity-management): correct what this test says the abandon would do The comment said the abandon would overwrite the operator's audited reason with an anonymous one. It would not — both exits prefix the existing message rather than replacing it, which I established while correcting the production comment in the previous commit and then failed to carry over to here, leaving the two saying opposite things about the same code. What actually differs is the verdict each exit adds: the release states the venue confirmed the request never arrived, the abandon only that nothing is left to execute, and just one of them rests on somebody having checked. That is what the branch order protects and what the assertion is worth having for. * fix(liquidity-management): give quarantined Scrypt withdrawals an automatic exit A quarantined withdrawal had no way out but an operator: cancelOutstanding returned false for WITHDRAW without asking anything, because Scrypt has no cancel operation for withdrawals. Trades resolved themselves, withdrawals waited for a person who may never come — and withdrawals are roughly half of what this venue does. The exit cannot rest on a cancellation, so it rests on confirmed absence: the venue returns its full transaction history and this reference is not in it. That is weaker than "the request never arrived" and is worded as such — the order is abandoned, never released as not-sent, because nobody observed a send. What carries the safety is the consistency gate, not the age bound. A fresh history counts only when it is non-empty and every already-known transaction inside the order's window reappears in it. Without that, a truncated reply that merely omits the reference would read as "does not exist", the rule would replan, and withdraw() would run again — it only checks minAmount > balance and then takes min(maxAmount, balance), so with enough balance left a second payout goes through. A failed gate warns with the missing references named, so a permanently broken fetch is visible instead of becoming a silent wait. cancelOutstanding now returns the reason to record instead of a boolean, so an abandoned withdrawal does not carry the trade wording claiming every reference was answered for. Reaching quarantine without a reference is an invariant break since reserve-before-send, not a planned wait: logged as an error rather than silently held. Verified against production — no Scrypt order is in Uncertain, and every one without a reference is Failed or NotProcessable, so there is no legacy population to migrate. Trades whose references the venue will not settle keep waiting, deliberately: that wait is on the venue, not on a person. While Scrypt is silent nothing can be planned against it anyway, and the next pass abandons the order within seconds of it answering. Releasing earlier would be guessing, and an onFail chain could then buy the same funds on a second venue while a zombie order still sits in the book. * fix(liquidity-management): close the remaining paths where Scrypt waits for a person Review found three more states a quarantined Scrypt order could only leave by hand, plus a gate that did not hold what it promised. The consistency gate was only as good as the local cache it compared against. With that cache empty — process restart, failed warm-up, catch-up not through yet — it passed without checking anything, so a truncated history that merely omitted the reference would have read as confirmed absence. It now requires an anchor: at least one known reference inside the window, plus the newest known transaction overall, so an empty time window cannot hollow it out either. Missing anchor means no exit and a warning, which only defers — warm-up refills the cache within seconds. The test that accepted the unsafe state now demands the safe one. An order whose command is no longer registered was skipped on every pass: the factory hands out an integration only for a supported command, and Scrypt/sell-if-deficit still exists in production as an action the code dropped. Reconciliation now resolves by system alone, because what matters for resolving a quarantined order is who can ask the venue, not whether the command is still executable. Executing still requires a registered command. Cancellation for such an order sends nothing — it asks after every reference and gives up only where the venue reports all of them terminal. An order without a reference was a dead end, needlessly: the reference is derived from the order id alone, so it can be reconstructed and the venue asked as usual. The invariant-break error stays, it is just no longer the only response. A missing `updated` switched the deadline off entirely, which turned a defensive guard into a permanent wait. It falls back to `created` — always older, so the bound expires earlier, which is safe because abandoning still needs the venue's confirmation; the clock only decides when a cleanup attempt is worth making. Without either timestamp the order still stays put, or the bound would run from the epoch and give up at once. A test asserted the old behaviour by name ("never abandons an order whose quarantine timestamp is missing") and was guarding the very defect; its claim is inverted rather than dropped. Two comments promised automation the code does not deliver for systems other than Scrypt. * fix(exchange): make the absence proof survive a venue that answers incompletely The consistency gate compared the fresh history against the local cache — but both come from the same client, and fetchAll treats a missing `next` as end-of-history without checking cursor or sequence. A truncated pagination therefore looks complete, and the same missing slice is absent from cache and reply alike, so the gate passed on a common-mode failure. The newest-known anchor did not help: it guards recency, while the failure drops an older row. An old anchor is added alongside it. The oldest cached reference stamped before the order's own window must reappear in the fresh reply, which forces the answer to span that window and exposes a suffix cut in either direction. What this cannot see is stated where the check lives rather than glossed over: a hole in the middle of history, missing from cache and reply alike, has nothing local to be compared against. The cache snapshot was taken before the fetch and never re-read, so a withdrawal arriving as a live push while the bulk fetch was open was invisible to both snapshot and reply — and would have been reported absent. The live map is now re-read after the await. The anchor requirement introduced a way to wait forever: the cache is filled by the boot warm-up and by reconnects only, and a failed warm-up had no retry. On a stable socket the cache would then stay empty, every reconciliation would defer, and the withdrawal would sit there — the forbidden combination, reintroduced by its own fix. A warm-up that fails now triggers the existing catch-up instead of waiting for a reconnect, and only a real rejection counts: an empty history is a successful warm-up. Cancelling for a command that is no longer registered was passive, waiting for a terminal status that may never come. The premise was wrong: getTradePair matches a pair in either direction and returns the same symbol, and a cancel reads only that symbol, never the side. With a tradeAsset in the params such an order takes the ordinary cancel path and gains the UnknownOrder inference with it. Only without a determinable symbol does the passive branch remain. * fix(exchange): drop the absence gate, it bought a non-risk with a forbidden wait The gate guarded against a second withdrawal. That is not a risk worth guarding: Scrypt withdrawal destinations are exclusively DFX-owned addresses, so a repeated payout moves funds between our own accounts — an internal rebooking, not a loss and not a compliance incident. What it cost instead was the one thing that is not allowed. Every anchor it required can be structurally absent: no cached row carrying a ClReqID, no row older than the order, nothing inside the order's window, or simply an order older than the 365-day cache bound. In each of those cases the check returned false on every pass, forever, and the withdrawal waited for someone to refill a cache by hand. Review found two such paths; both came from the gate itself. So the gate goes, and with it roughly 120 lines. What remains reacts to an absent answer rather than to a missing local anchor — a fetch that throws, or an empty history — plus the one positive observation that costs nothing: if the reference turns up in the live cache while the bulk fetch is open, absence is not confirmed. A truncated venue reply can now cause a second withdrawal, which is written where the check lives, because it is a decision and not an oversight. Two defects surfaced underneath and are fixed on their own merits. Balance transactions without a readable timestamp were dropped from the cache unconditionally — `new Date(undefined) >= x` is always false — while the sibling function five lines above guards exactly that case; the rows discarded were withdrawals we later look for. And the passive path for an unregistered command treated "the venue does not know this reference" like "the venue could not be asked", so it never settled; the active path infers precisely that. Only a lookup that could not be performed keeps the order waiting now. * fix(exchange): close the last two states a Scrypt order could not leave alone An empty transaction history was treated as an answer we could not use. It is the opposite: on a fresh or long-idle account — the stream carries only deposits and withdrawals, never trades — the venue answers successfully with zero rows, and a reference cannot be in a list that has none. The branch turned that into a permanent wait, and the warm-up retry cannot break it, because an empty result counts as a *successful* warm-up. Two of my own comments contradicted each other on exactly this point; the one at the check claimed a total integration failure that recovery would repair, three lines from the constructor saying an empty array triggers nothing. A real outage throws and is caught above, so the two similar-looking cases are handled differently on purpose. That is now written where the check is. The second state was a trade of a command the code no longer has. Cancelling needs a symbol, the order carried none that could be trusted, and Scrypt orders run until cancelled — so nothing would ever end it. The premise was wrong: ScryptOrderInfo already names the symbol a reference lives under, and a cancel reads nothing but that symbol. The venue therefore tells us how to cancel its own order, and every reference it can still show is cancellable. So the symbol derivation goes. It was not only unnecessary but unsound: a tradeAsset left in a stale paramMap is no evidence of what the command once did, "SELL versus everything else" is not a side inference for a command that by definition is not SELL, and the pair lookup reads live security configuration that a delisted pair no longer matches — while the order sits open at the venue under its original symbol. One path now covers every unsupported command: ask, then cancel under the symbol the venue hands back, evaluated exactly as the known-command path evaluates a cancel. `null` still means the venue has no record; only `undefined` — could not be asked — keeps the order waiting, and waiting on the venue is allowed. cancelIfOutstanding keeps its behaviour byte for byte; it and the new symbol-taking variant share one core rather than a second copy of the guards. * fix(liquidity-management): do not advance an order past the check that would finish it Making reconciliation resolve by system alone let an unregistered command be observed again — that part was right and stays. What it also did was let such an order back into IN_PROGRESS on a venue-confirmed SENT, and that was a trap of my own making. The normal completion path uses the strict lookup, which returns nothing for an unregistered command, and the result was dereferenced unchecked: every pass threw a TypeError that the surrounding catch merely logged. The order sat in IN_PROGRESS, where the automatic abandon path does not reach and the manual endpoint refuses it for no longer being quarantined. Worse than the state it replaced, because quarantine at least has an exit. Taking the loose lookup into the completion path instead would not have helped: checkCompletion answers `false` for a command it does not know, so the order would hang just as long without even an error to show for it. The exit has to run through quarantine, which is where the cancel path now works. So the rule is split by what a caller can actually do. Observing is allowed for anyone who can ask the venue — reconciliation stays command-independent. Advancing out of quarantine requires whoever can also finish the job, so a venue-side SENT for a vanished command stays quarantined and leaves by the ordinary cancel/abandon route. Second layer, for any system rather than this one: a running order whose adapter is gone now raises an unknown-outcome error instead of a TypeError, so the existing handler quarantines it. That turns a silent permanent loop into the state from which both the automatic and the manual exit apply. The docstring claimed a confirmed order goes back to IN_PROGRESS and the normal check takes over, three lines above a comment describing the removed-command case as covered. Both were true separately and wrong together; they now say the same thing. No order in production is affected — Created, InProgress and Uncertain are all empty. The defect is structural and would have surfaced at the next rename. * fix(liquidity-management): make the cleanup reachable from the branch that needs it The previous commit kept an order quarantined when its command was no longer registered, so it would not fall into the trap of an IN_PROGRESS state nothing could finish. It also cut off the exit it promised. Resolution runs as a single if/else-if chain, the guard sat in the first branch, and `cancelOutstanding` has exactly one call site — in the last one. A venue-confirmed SENT therefore won the chain every pass and the age check was never evaluated. One dead end traded for another. Worse than it first looks, because the state is stable rather than transient: resolveUncertainOrder answers whether the reference exists, not whether it is still open, so a reference the venue knows returns SENT forever — including one long since filled or cancelled there. An exit conditioned on that answer changing never arrives. The cleanup — bound check, cancel, abandon — moves into its own method and is now called from both branches. Deliberately not by reordering the chain: a case falling further through it would meet the release branches, and with an operator release pending that writes a not-sent reason onto the order. After the venue has just confirmed the reference, that reason would be false. The tests are the other half of this. The previous one asserted only the status after a single pass, which is why 301 green tests hid a severed exit. They now assert the mechanism: past the bound the cancel must be attempted, inside it must not, an unsettled cancel keeps the order waiting on the venue, and a released order past its bound leaves through the cancel rather than through a not-sent release that would claim the request never arrived. * docs(liquidity-management): stop promising a completeness check that was removed Three files still described the withdrawal exit as resting on a "complete, consistent transaction history", and one named a null-return for an incomplete one. That check existed for two commits and was deliberately deleted: it demanded local cache anchors that can be structurally absent, so it stranded the very orders it was meant to protect, while the risk it guarded — a repeat payout to a DFX-owned address — is an internal rebooking. The code says so where the check lives; these three did not, and a comment promising a guarantee the code does not give is the same defect as the guarantee being missing. They now say what actually happens: the venue answered and did not name the reference, with no completeness check on that answer, weaker than "never arrived" and weaker than "the history was whole". The interface doc also had it backwards for unsupported commands, claiming no cancel is sent because no symbol can be derived. That stopped being true when the symbol started coming from the venue's own order-status reply. * fix(liquidity-management): stop the recorded reason from claiming a complete history The reason string written onto an abandoned withdrawal said the venue "returned its full transaction history". It does not: the reply is taken as it comes, without a completeness check, which is the trade-off argued where the check lives. This string outlives the log — it lands in the order and is what someone reads months later to understand why the order ended — so it was the worst place for that claim to survive. It now says the venue answered and did not name the withdrawal, which is exactly what was observed. The same overclaim sat in four comments describing that exit, and one paragraph in the interface doc still promised a "full history" four lines under a sentence that had just been corrected to deny it. One more correction is not about promising too much but about describing the wrong thing: the null cases were summarised as "a failed or empty-of-answer lookup". Both halves were wrong. An empty reply confirms absence rather than blocking it, and the case actually left out is the one that matters most — the reference still being named in the reply. Null now lists all three: the lookup failed, the reply names the reference, or it surfaced in the live cache while the lookup was open. * docs(liquidity-management): match the last test comment to the corrected wording The phrase describing the withdrawal exit as an absence from a full history was replaced everywhere else; this one sat in a test file the previous commit did not touch. * fix(liquidity-management): cap self-resolution at ten minutes for Scrypt Twelve hours was too long. The ceiling is now ten minutes end to end: five before a withdrawal with no venue record at all counts as doubtful, and five more before the automatic exit is attempted. Trades already fit — measured over 60 days they complete in a median of 0.2 and at most 1.0 minutes. The transfer bound is not lowered globally. It also governs bridges, mints and other systems' transfers, and none of those adapters implements cancelOutstanding, so no exit would be attempted there anyway — the only effect would be asking their venues more often for nothing. Scrypt withdrawals get their own category instead, on the same allowlist discipline as the trade bound: anything unrecognised keeps the long one. The short value is deliberately not derived from how long withdrawals take. Measured, they run to 336 minutes and 39% of them past ten. That tail does not reach this bound, because the bound only applies while the venue does not name the reference — one it does name is SENT and runs to completion on its own clock. What the short value costs is the case where the venue accepted a withdrawal and publishes it late: it is given up and reissued. Every Scrypt withdrawal address is DFX-owned, so that is an internal rebooking, which is the same ground on which the absence check itself was already accepted. The unobservable window drops from sixty minutes to five for the same ceiling, and ORDER_LOST_AFTER_MINUTES follows it: the adapter's doc names that value as the one it matches, and the two are the pair of routes out of a silent order, so they must not drift apart. Safe at five because it only fires while the venue does not know the reference at all — an order it is still working comes back from the status lookup whatever its age. Four tests asserted the old split by name ("holds a withdrawal far longer than a trade — its p95 alone is over an hour") or used a Scrypt withdrawal as their stand-in for a long bound while measuring the cooldown. The first now states what actually holds, the others moved to a venue that cannot be asked, which is where the long bound still lives. * docs(exchange): correct what the lost-order bound applies to, and how exact ten minutes is Two review findings, both mine to answer for. The comment above ORDER_LOST_AFTER_MINUTES claimed the value is shared by the "cannot be found" and the "stuck pending" paths. It is not — there is one use, the branch where the status lookup returns nothing at all. The pending states wait however old the order is, deliberately: a venue still reporting PENDING_NEW is answering, and this constant is about silence. The claim predates this branch, but I extended that comment and carried it along, which is worse than leaving it alone. The line below it still read "older than 1 hour" after the value became five minutes. And the ceiling is nominal. The pass runs on a ten-second cron with a few seconds of jitter, and a deadline missed just after a tick waits out the one-minute cooldown floor, so ten minutes is really ten to eleven plus the venue round-trip. Stated where the bound is defined, because "maximum ten minutes" is the kind of number that gets quoted back later. * fix(liquidity-management): bound the two Scrypt states only a person could end Review of the final state found two more places where an order could only ever be ended by someone noticing it. Both were missed for the same reason: the venue keeps answering, so neither looked like waiting. A trade the venue reports as PENDING_NEW/CANCEL/REPLACE waited without any limit, on the argument that a pending report is an observation rather than an unknown outcome. That argument holds against quarantining it — reconciliation would hand it straight back and the next check would quarantine it again — but not against waiting forever: "observed" is not the same claim as "running", and the only exit left was a monitoring counter and then a person. PENDING_* is meant to last seconds, and measured trades take a median of 0.2 and at most 1.0 minutes. Past five minutes the check now asks the venue to cancel. Only a confirmed cancel ends the order; an unconfirmed one keeps waiting, because a trade that might still be sitting in the book must not be given up while an onFail chain could place a genuinely competing buy beside it. A withdrawal the venue reports without a transaction hash had the same shape — only the complete absence of a record was bounded. Withdrawals take a median of 6.6 minutes, 90 at p95 and 336 at the slowest observed run, so a 24-hour backstop is four times that worst case, far outside what slowness explains. Deliberately not the ten-minute bound, which belongs to quarantine, and deliberately not routed through quarantine either: a record already on file makes the absence proof unreachable, so the order would only bounce between reconciliation and this check. It fails the order so the rule replans; a payout landing afterwards is an internal rebooking, every Scrypt withdrawal address being ours. Three comment corrections from the same review, all of them mine to answer for. The adapter claimed the ten-minute ceiling unconditionally, where it only holds if the venue answers at all. The reason string written to errorMessage said the venue reported every reference in a terminal state, while the same loop treats one the venue does not know at all exactly the same way — that is an inference from its silence about the reference, not a status it reported. And an empty history reply was called a complete statement of absence four lines below the paragraph that admits a truncated reply can arrive looking just like one. The bound for Scrypt withdrawals also had no test that could tell it apart from the twelve-hour transfer bound: the existing one used 780 minutes, past both. A typo in the allowlist would have moved every withdrawal to the long bound unnoticed, which is the ten-minute promise itself. Verified by mutation — misspelling the entry turns the new test red. * fix(exchange): throttle the pending cancel, and say only what the venue said Six findings from the review of the previous commit, all of them mine. The cancel that ends a stuck pending trade is a WRITE, and checkRunningOrders drives it from an ungated ten-second cron. An UNCONFIRMED answer therefore drew a fresh cancel every ten seconds, per order, for as long as the venue kept not confirming — the exact behaviour this branch forbids elsewhere in the same PR ("a cancellation the venue will not confirm must not retry on every ten-second tick"), and worst precisely during the venue trouble that produces UNCONFIRMED in the first place. There is now a one-minute floor per reference, matching the quarantine path's cooldown. It slows the write down without moving the bound: an order becomes eligible to be cancelled at the same age either way. Verified by mutation — dropping the floor to zero turns the new test red. A missing orderCreated read as age zero and quietly exempted the order from the bound. That is the admin trade endpoint, which tracks trades in memory for a person who started them and polls the result, not a liquidity order that could block a rule unseen — but a silent fallback is the wrong way to say so. It is now its own branch, named and explained, with a test. Three of the six are claims that outrun what the code establishes: - The new error class documented the venue's answer as "a verdict, not a gap". SETTLED covers two qualities, and the second — the venue not knowing the reference — is an inference from its own words. It happens to hold here, because the venue reported that same reference as PENDING moments earlier and both readings then lead to the same conclusion, but that is an argument to make, not to skip. - The withdrawal backstop said the venue "reports it in progress". It cannot: ScryptTransactionStatus is Completed | Failed | Rejected, and the last two are already caught above. What the branch actually knows is that there is no transaction hash. It now reports the status the venue really sent. - A bound comment still promised a named withdrawal "runs to completion on its own clock, however long that takes", which the 24-hour backstop ended. Also corrected: the stuck-pending message reported the order's total age as though it were time spent in that status. * fix(exchange): require the trade's creation time, and finish the corrections Third review pass. Most of it is me not carrying my own corrections through. The previous commit gave a missing orderCreated its own waiting branch and justified it with the admin trade endpoint. That justification was wrong: the controller only ever resolves through the registry's `get`, and ScryptService never registers there — for Scrypt there is a separate accessor the controller does not use. The branch was unreachable and its comment invented a caller. So rather than correcting the comment, the parameter is now required. That removes the unreachable branch, the invented caller, and the same silent fallback twenty lines above it, where a missing timestamp read as age zero. Every actual caller already passes order.created, from a non-optional column. Its test goes too — it described a state that can no longer exist. The "verdict, not an observation gap" wording was softened at the source last commit but left standing verbatim at the catch site, which is where the order actually fails. It now points at the error class and names what the settlement rests on: a terminal cancel, or the venue no longer recognising the reference, which is an inference from its own contradiction. Both readings still land in the same place; that is the argument, and it belongs where the decision is made. The same JSDoc claimed the venue had "just" reported the reference as PENDING. It comes from the cached execution report, not re-fetched, and this branch is only reached after five minutes — so the observation can be minutes old. The contradiction argument holds either way, but not on the strength of that word. And "ungated 10-second cron" ignored the cron's lock, its jitter lead-in, and that processPipelines' own loop can invoke the check again before the next tick. Two robustness points from the same pass: - The retry stamp now sits in a `finally`. Today cancelIfOutstandingCore swallows everything into UNCONFIRMED so the throw cannot happen, but a dead connection is exactly the regime the floor exists for, and the quarantine throttle stamps in a finally for that stated reason. - The throttle test only proved the cancel does not repeat too soon. That an implementation whose floor never expires would look identical to a correct one went unasserted — and a permanently blocked cancel is itself a new stuck-forever state. A second test advances past the floor and asserts the retry happens. Verified by mutation: making the floor never expire turns it red. * docs(exchange): drop two claims the pending bound cannot support Fourth review pass, comments only — no behaviour change. The stuck-pending error argued that a venue answering UnknownOrder contradicts its own last answer, since it had reported the reference as PENDING. It cannot carry that: the status read there comes from the cached execution report and is not re-fetched before the cancel, so a live push may have moved it on in between. "Its last answer" is then not what the branch saw. The conclusion is unchanged and rests where it always did — on the inference documented at SCRYPT_UNKNOWN_ORDER, that a venue not recognising a reference cannot execute anything under it. The mirrored comment at the catch site says the same. And the bound was described as a trade reporting PENDING for longer than that transition should take. It measures the order's age, not its dwell time in the state: a PENDING_CANCEL may have begun seconds ago on an order open for hours. The venue reports a status, not how long it has held it, so dwell time is not available to measure against. Age is the conservative substitute — it can only reach the bound later than a true dwell-time clock would, never earlier, which is why it is a safe stand-in and worth saying rather than glossing. * fix(exchange): measure the pending bound against dwell time, not order age Review caught this in a comment I had just written to fix a different imprecision, and it turned out to be a real defect rather than wording. The five-minute pending bound compared PENDING_STUCK_AFTER_MINUTES against the order's age. With t0 for creation and t1 for entering a pending state, t1 >= t0 always, so age >= dwell time: the age clock reaches the bound earlier or at the same moment, never later. The comment claimed the exact opposite, and the example it used to illustrate the point refuted it. What that cost in practice: an order running healthily for two hours that briefly reports PENDING_REPLACE — a price adjustment, which this code issues itself — is past a five-minute age bound on the very first poll and gets cancelled, instead of being given the seconds such a transition takes. So the bound now measures what it is about. A map records when a reference was first seen pending, the wait is counted from there, and the entry is cleared as soon as the venue reports any non-pending status, so a later re-entry starts a fresh observation rather than inheriting an expired one. A process restart begins the count again, which only ever lengthens the wait and never gives up early. Pruned past 24 hours, like the retry map beside it. The SCRYPT_UNKNOWN_ORDER inference also needed its premise widened. It was written for a caller whose status lookup had already failed, and stated that as the ground it rests on. The pending path's lookup succeeded — it returned PENDING — so the comment no longer covered its second caller. For that one the premise is in fact stronger: the reference demonstrably existed, and the venue then did not recognise it on the cancel. The regression test needed a second pass to be worth anything. Asserting the first pending poll proves nothing: it only records the observation and returns, whichever clock the bound uses. Verified by mutation — restoring the age comparison leaves the single-pass version green and turns the two-pass version red. * fix(exchange): prune the pending clock by last sighting, not by its start Three findings from the sixth review pass. The pending map stored one timestamp and used it for two jobs. `since` is the clock the bound measures against, so it is set once and never moved. The 24-hour sweep then read that same value as though it meant "last seen", and its comment claimed an entry that old belongs to a reference that has long since moved on. The code itself disproves that: a reference whose cancel keeps coming back UNCONFIRMED stays pending and keeps being polled indefinitely, by design. Such an entry would have been swept away by the next unrelated reference entering pending, and the stuck order would have been handed a fresh five-minute grace period instead of reaching its bound. The two meanings are now separate fields — `since` still measures, `lastSeen` is written on every sighting and is what the sweep reads. The comment is true as written now. That needed a test of its own: the case only arises after a day, so nothing covered it. Restoring the old sweep now turns one red, which is how I know it covers something. Mutation-checked in both directions. The JSDoc on ScryptOrderStuckPendingError still carried the age argument the previous commit removed from the service — a third copy of the same reasoning, in another file, now contradicting the behaviour outright. Corrected to what the bound actually measures. And the test asserting that an unconfirmed cancel keeps the order waiting only checked the return value. It could not tell "waited because the cancel was not confirmed" from "waited because no cancel was ever attempted" — the second is what a regression would look like. It now holds the spy and asserts the attempt. * test(exchange): assert the pending clock is refreshed, not just read The sweep test proved only half its own claim. It showed that cleanup reads lastSeen rather than since, which was the fix — but never took the branch that refreshes lastSeen, because it observed the reference only once. Delete that refresh outright and the suite stayed green: the sweep would still find the stamp written when the entry was created, while a real reference would quietly age into being collectable. The test now observes the reference a second time with both stamps aged past a day, and asserts what that pass must do: move lastSeen forward, and leave since where it is. The second half matters as much as the first — pushing the clock along on every sighting would mean the bound is never reached at all. --- src/integration/exchange/dto/scrypt.dto.ts | 40 + .../services/__tests__/scrypt.service.spec.ts | 754 +++++++++++++++++- .../services/scrypt-websocket-connection.ts | 25 + .../exchange/services/scrypt.service.ts | 452 ++++++++++- .../actions/__tests__/scrypt.adapter.spec.ts | 306 ++++++- .../adapters/actions/scrypt.adapter.ts | 316 +++++++- .../liquidity-management-order.entity.spec.ts | 174 ++++ .../liquidity-management-order.entity.ts | 219 ++++- .../core/liquidity-management/enums/index.ts | 22 +- .../liquidity-action-integration.factory.ts | 13 + .../liquidity-management/interfaces/index.ts | 30 + ...uidity-management-pipeline.service.spec.ts | 586 +++++++++++++- .../liquidity-management-pipeline.service.ts | 235 +++++- 13 files changed, 3080 insertions(+), 92 deletions(-) diff --git a/src/integration/exchange/dto/scrypt.dto.ts b/src/integration/exchange/dto/scrypt.dto.ts index 4905bd82a8..01eccd7e00 100644 --- a/src/integration/exchange/dto/scrypt.dto.ts +++ b/src/integration/exchange/dto/scrypt.dto.ts @@ -107,11 +107,51 @@ export enum ScryptOrderStatus { PENDING_REPLACE = 'PendingReplace', } +/** + * Terminal order statuses at Scrypt: nothing under the reference can still execute. + * + * Single source of truth for both execution-report caching (`OrdStatus`) and order-status lookups + * (`ScryptOrderInfo.status`). Do not re-list these three values elsewhere. + */ +export const SCRYPT_TERMINAL_ORDER_STATUSES: readonly ScryptOrderStatus[] = [ + ScryptOrderStatus.FILLED, + ScryptOrderStatus.CANCELED, + ScryptOrderStatus.REJECTED, +]; + +export function isTerminalScryptOrderStatus(status: ScryptOrderStatus): boolean { + return (SCRYPT_TERMINAL_ORDER_STATUSES as readonly ScryptOrderStatus[]).includes(status); +} + export enum ScryptOrderSide { BUY = 'Buy', SELL = 'Sell', } +/** + * What asking the venue to cancel a reference established about it. + * + * Three outcomes, not two, because "cancelled" does not mean "nothing happened": a partially filled order + * is cancelled with a terminal status AND a fill, and that fill is worth naming rather than folding into + * the same answer as an untouched one. + */ +export enum ScryptCancellation { + /** + * Nothing can execute under this reference any more. Two different qualities of answer: cancelled with + * nothing filled settles it outright, while the venue not knowing the reference is an inference from its + * own words — see SCRYPT_UNKNOWN_ORDER for what that evidence covers. + */ + SETTLED = 'Settled', + /** + * It reached a terminal state with something filled. Like a cancelled reference it cannot trade further, + * so the order may be given up — the fill already moved the venue balance the rule replans from. Kept + * distinct from SETTLED because a fill is worth seeing in a log and worth reconciling against. + */ + EXECUTED = 'Executed', + /** No usable answer. Nothing may be concluded, least of all that the reference is safe to walk away from. */ + UNCONFIRMED = 'Unconfirmed', +} + export enum ScryptOrderType { MARKET = 'Market', LIMIT = 'Limit', diff --git a/src/integration/exchange/services/__tests__/scrypt.service.spec.ts b/src/integration/exchange/services/__tests__/scrypt.service.spec.ts index 1a24960be8..1afa09e800 100644 --- a/src/integration/exchange/services/__tests__/scrypt.service.spec.ts +++ b/src/integration/exchange/services/__tests__/scrypt.service.spec.ts @@ -2,6 +2,8 @@ import { GetConfig } from 'src/config/config'; import { Util } from 'src/shared/utils/util'; import { ScryptBalanceTransaction, + ScryptCancellation, + ScryptExecutionReport, ScryptOrderStatus, ScryptTransactionStatus, ScryptTransactionType, @@ -9,6 +11,7 @@ import { import { ScryptAmendRejectedError, ScryptMessageType, + ScryptOrderStuckPendingError, ScryptRequestTimeoutError, ScryptUnconfirmedWriteError, ScryptVenueRejectionError, @@ -361,15 +364,20 @@ describe('ScryptService', () => { expect((service as any).lastCatchUpAt).toBeGreaterThan(startStamp!); }); - it('a warm-up that failed does not claim the catch-up slot', async () => { + it('a warm-up that failed schedules catch-up without waiting for a reconnect', async () => { + // After FIX 2 a failed warm-up immediately reuses catchUpAfterReconnect. That stamps lastCatchUpAt at + // the end of every round (pacing, independent of success) and arms catchUpRetryTimer when legs stay + // owed — so lastCatchUpAt is no longer undefined after a failed warm-up. The invariant that matters is + // that a retry is scheduled without onReconnect ever firing the registered callback. const MockedConnection = ScryptWebSocketConnection as jest.MockedClass; + const onReconnect = jest.fn(); MockedConnection.mockImplementationOnce( () => ({ fetchAll: jest.fn().mockRejectedValue(new Error('Connection closed')), fetch: jest.fn().mockResolvedValue([]), subscribeToStream: jest.fn().mockReturnValue(() => undefined), - onReconnect: jest.fn(), + onReconnect, send: jest.fn(), requestAndWaitForUpdate: jest.fn(), }) as any, @@ -377,10 +385,57 @@ describe('ScryptService', () => { const freshService = new ScryptService(); jest.spyOn((freshService as any).logger, 'error').mockImplementation(() => undefined); + jest.spyOn((freshService as any).logger, 'warn').mockImplementation(() => undefined); await flushPromises(); - // The caches are not whole, so the next reconnect must repair immediately instead of sitting out the interval. - expect((freshService as any).lastCatchUpAt).toBeUndefined(); + // onReconnect only registers the handler; we never invoke that callback. The timer proves the + // boot-failure path armed a retry on its own without a reconnect. + expect(onReconnect).toHaveBeenCalled(); + expect(typeof onReconnect.mock.calls[0][0]).toBe('function'); + expect((freshService as any).catchUpRetryTimer).toBeDefined(); + (freshService as any).clearCatchUpRetry(); + }); + + it('a failed boot warm-up retries catch-up without a reconnect and respects catchUpMinInterval', async () => { + // FIX 2: when warm-up rejects and the socket stays up, catchUpAfterReconnect must run immediately and + // its scheduled retry must actually re-enter after catchUpMinInterval — otherwise the empty cache is a + // permanent deferral (anchor check forever false) rather than a temporary one. + jest.useFakeTimers(); + const MockedConnection = ScryptWebSocketConnection as jest.MockedClass; + const fetchAll = jest.fn().mockRejectedValue(new Error('Connection closed')); + const onReconnect = jest.fn(); + MockedConnection.mockImplementationOnce( + () => + ({ + fetchAll, + fetch: jest.fn().mockResolvedValue([]), + subscribeToStream: jest.fn().mockReturnValue(() => undefined), + onReconnect, + send: jest.fn(), + requestAndWaitForUpdate: jest.fn(), + }) as any, + ); + + const freshService = new ScryptService(); + jest.spyOn((freshService as any).logger, 'error').mockImplementation(() => undefined); + jest.spyOn((freshService as any).logger, 'warn').mockImplementation(() => undefined); + await flushPromises(); + + // (1) Retry armed without ever invoking the reconnect callback registered via onReconnect. + expect((freshService as any).catchUpRetryTimer).toBeDefined(); + const reconnectHandler = onReconnect.mock.calls[0]?.[0] as (() => void) | undefined; + // Handler is registered; we never call it — refill must not depend on a drop. + expect(typeof reconnectHandler).toBe('function'); + + // (2) The armed retry is not cosmetic: after catchUpMinInterval further fetchAll calls land. + const callsBeforeRetry = fetchAll.mock.calls.length; + expect(callsBeforeRetry).toBeGreaterThan(0); + jest.advanceTimersByTime((freshService as any).catchUpMinInterval); + await flushPromises(); + expect(fetchAll.mock.calls.length).toBeGreaterThan(callsBeforeRetry); + + (freshService as any).clearCatchUpRetry(); + jest.useRealTimers(); }); it('a warm-up that loaded both streams claims the catch-up slot', async () => { @@ -677,6 +732,72 @@ describe('ScryptService', () => { expect((freshService as any).balanceTransactions.get('warm-old')).toBeUndefined(); }); + it('bulk applyBalanceTransactions caches rows without Timestamp when TransactTime is set', async () => { + // Field priority: Timestamp missing → TransactTime; recent stamp must still land in the cache. + const recent = new Date().toISOString(); + const noTimestamp = { + ClReqID: 'warm-transact-time', + TransactionID: 'tx-warm-tt', + Status: ScryptTransactionStatus.COMPLETED, + TransactTime: recent, + }; + + const MockedConnection = ScryptWebSocketConnection as jest.MockedClass; + MockedConnection.mockImplementationOnce( + () => + ({ + fetchAll: jest.fn().mockImplementation(async (streamName: string) => { + if (streamName === ScryptMessageType.BALANCE_TRANSACTION) { + return [noTimestamp]; + } + return []; + }), + fetch: jest.fn().mockResolvedValue([]), + subscribeToStream: jest.fn().mockReturnValue(() => undefined), + onReconnect: jest.fn(), + send: jest.fn(), + requestAndWaitForUpdate: jest.fn(), + }) as any, + ); + + const freshService = new ScryptService(); + await flushPromises(); + + expect((freshService as any).balanceTransactions.get('warm-transact-time')).toEqual(noTimestamp); + }); + + it('bulk applyBalanceTransactions caches rows with neither Timestamp nor TransactTime (conservative)', async () => { + // Missing/unreadable stamp must not drop a withdrawal we later need for findWithdrawal / live recheck. + const noStamp = { + ClReqID: 'warm-no-stamp', + TransactionID: 'tx-warm-no-stamp', + Status: ScryptTransactionStatus.COMPLETED, + }; + + const MockedConnection = ScryptWebSocketConnection as jest.MockedClass; + MockedConnection.mockImplementationOnce( + () => + ({ + fetchAll: jest.fn().mockImplementation(async (streamName: string) => { + if (streamName === ScryptMessageType.BALANCE_TRANSACTION) { + return [noStamp]; + } + return []; + }), + fetch: jest.fn().mockResolvedValue([]), + subscribeToStream: jest.fn().mockReturnValue(() => undefined), + onReconnect: jest.fn(), + send: jest.fn(), + requestAndWaitForUpdate: jest.fn(), + }) as any, + ); + + const freshService = new ScryptService(); + await flushPromises(); + + expect((freshService as any).balanceTransactions.get('warm-no-stamp')).toEqual(noStamp); + }); + it('catchUpAfterReconnect coalesces a reconnect seen during the fetches into one follow-up round', async () => { let fetchAllCallCount = 0; @@ -806,6 +927,399 @@ describe('ScryptService', () => { ]); }); }); + describe('confirmWithdrawalAbsent', () => { + const soughtId = 'dfx-lm-withdraw-9'; + + function balanceTx(overrides: Partial = {}): ScryptBalanceTransaction { + return { + TransactionID: 'tx-1', + ClReqID: 'other-ref', + Currency: 'CHF', + TransactionType: ScryptTransactionType.WITHDRAWAL, + Status: ScryptTransactionStatus.COMPLETED, + Quantity: '100', + Timestamp: '2026-07-15T12:00:00.000Z', + ...overrides, + }; + } + + it('returns true when the cache is empty and the sought reference is absent from a non-empty fresh history', async () => { + // Empty process cache is no longer a reason to refuse: absence is decided from the fresh venue reply + // (plus the live recheck). No local anchor is required. + instance.fetchAll.mockResolvedValue([balanceTx({ ClReqID: 'unrelated-in-history', TransactionID: 'tx-u' })]); + + await expect(service.confirmWithdrawalAbsent(soughtId)).resolves.toBe(true); + }); + + it('returns true when the cache holds only post-since rows and the sought reference is absent from fresh', async () => { + // Cache rows newer than the order no longer act as consistency anchors; missing sought id → true. + (service as any).balanceTransactions.set('recent-only', { + ...balanceTx({ ClReqID: 'recent-only', Timestamp: '2026-07-20T00:00:00.000Z' }), + }); + instance.fetchAll.mockResolvedValue([ + balanceTx({ ClReqID: 'recent-only', TransactionID: 'tx-r', Timestamp: '2026-07-20T00:00:00.000Z' }), + balanceTx({ ClReqID: 'unrelated-in-history', TransactionID: 'tx-u', Timestamp: '2026-07-21T00:00:00.000Z' }), + ]); + + await expect(service.confirmWithdrawalAbsent(soughtId)).resolves.toBe(true); + }); + + it('returns false when the fresh history contains the sought reference', async () => { + instance.fetchAll.mockResolvedValue([balanceTx({ ClReqID: soughtId, TransactionID: 'tx-sought' })]); + + await expect(service.confirmWithdrawalAbsent(soughtId)).resolves.toBe(false); + }); + + it('returns false and warns when the history fetch fails', async () => { + const warnSpy = jest.spyOn(service['logger'], 'warn').mockImplementation(); + instance.fetchAll.mockRejectedValue(new Error('Connection closed')); + + await expect(service.confirmWithdrawalAbsent(soughtId)).resolves.toBe(false); + expect(warnSpy).toHaveBeenCalled(); + }); + + it('returns true when the venue returns an empty history — no rows at all is a complete answer, not a reason to wait', async () => { + const infoSpy = jest.spyOn(service['logger'], 'info').mockImplementation(); + instance.fetchAll.mockResolvedValue([]); + + await expect(service.confirmWithdrawalAbsent(soughtId)).resolves.toBe(true); + expect(infoSpy).toHaveBeenCalled(); + }); + + it('returns false when the sought reference appears in the live cache while the history fetch is in flight', async () => { + // Live push mid-flight lands in this.balanceTransactions; freshIds does not include soughtId. Only the + // post-await re-read of the live map can block absence confirmation for that race. + const warnSpy = jest.spyOn(service['logger'], 'warn').mockImplementation(); + + instance.fetchAll.mockImplementationOnce(async () => { + (service as any).cacheBalanceTransaction( + balanceTx({ ClReqID: soughtId, TransactionID: 'tx-live', Timestamp: '2026-07-10T00:00:00.000Z' }), + ); + return [balanceTx({ ClReqID: 'unrelated', TransactionID: 'tx-u', Timestamp: '2026-07-20T00:00:00.000Z' })]; + }); + + await expect(service.confirmWithdrawalAbsent(soughtId)).resolves.toBe(false); + expect(warnSpy).toHaveBeenCalled(); + expect(warnSpy.mock.calls.some((c) => String(c[0]).includes(soughtId))).toBe(true); + }); + + it('returns true when the sought reference is absent from a non-empty fresh history', async () => { + // Cache content is irrelevant for absence confirmation as long as the live recheck does not hit. + instance.fetchAll.mockResolvedValue([ + balanceTx({ ClReqID: 'unrelated-in-history', TransactionID: 'tx-u', Timestamp: '2026-07-21T00:00:00.000Z' }), + ]); + + await expect(service.confirmWithdrawalAbsent(soughtId)).resolves.toBe(true); + }); + }); + + describe('cancelIfOutstanding', () => { + /** A complete report, so the fixture cannot drift from the contract cancelOrder now promises. */ + function cancelReport(overrides: Partial = {}): ScryptExecutionReport { + return { + ClOrdID: 'cancel-req-1', + OrigClOrdID: 'dfx-lm-7', + Symbol: 'EUR/USDT', + Side: 'Sell', + OrdStatus: ScryptOrderStatus.CANCELED, + OrderQty: '100', + CumQty: '0', + LeavesQty: '0', + ...overrides, + }; + } + + function stubCancel(report: ScryptExecutionReport | Error): void { + jest.spyOn(service as any, 'getTradePair').mockResolvedValue({ symbol: 'EUR/USDT' }); + const connection = (service as any).connection; + jest + .spyOn(connection, 'requestAndWaitForUpdate') + .mockImplementation(async () => (report instanceof Error ? Promise.reject(report) : report)); + } + + it('settles a cancellation that filled nothing', async () => { + stubCancel(cancelReport()); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe(ScryptCancellation.SETTLED); + }); + + it('reports a partial fill as executed — a terminal status does not mean nothing happened', async () => { + // the venue cancels a partially filled order with BOTH a terminal state and a non-zero filled size; + // reading only the state is how a real fill gets dropped + stubCancel(cancelReport({ CumQty: '40' })); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe(ScryptCancellation.EXECUTED); + }); + + it('settles a refusal that says there is no such order', async () => { + // a refused cancel arrives as an execution report, not as an error + stubCancel( + cancelReport({ OrdStatus: ScryptOrderStatus.NEW, ExecType: 'CancelRejected', CxlRejReason: 'UnknownOrder' }), + ); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe(ScryptCancellation.SETTLED); + }); + + it.each([ScryptOrderStatus.CANCELED, ScryptOrderStatus.FILLED])( + 'settles nothing when a %s report claims the order is unknown yet reports a fill', + async (ordStatus) => { + // the contradiction is the same whatever status rides along; deciding on the status first would let + // it through with a terminal one attached + stubCancel( + cancelReport({ + OrdStatus: ordStatus, + ExecType: 'CancelRejected', + CxlRejReason: 'UnknownOrder', + CumQty: '40', + }), + ); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe( + ScryptCancellation.UNCONFIRMED, + ); + }, + ); + + it.each(['', ' ', 'abc', undefined])('does not cache a cancellation whose filled size is %p', async (cumQty) => { + // readers derive the fill with `parseFloat(...) || 0`, so such an entry would quietly claim nothing + // was filled on every later lookup + stubCancel(cancelReport({ CumQty: cumQty as unknown as string })); + + await service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT'); + + expect((service as any).executionReports.has('dfx-lm-7')).toBe(false); + }); + + it('settles nothing when a refusal claims the order is unknown yet reports a fill', async () => { + // an order the venue has no record of cannot have traded — the report disagrees with itself, and + // nothing may be concluded from that, least of all that walking away is safe + stubCancel( + cancelReport({ + OrdStatus: ScryptOrderStatus.PARTIALLY_FILLED, + ExecType: 'CancelRejected', + CxlRejReason: 'UnknownOrder', + CumQty: '40', + LeavesQty: '60', + }), + ); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe( + ScryptCancellation.UNCONFIRMED, + ); + }); + + it('settles nothing on any other refusal — too late to cancel means it may yet execute', async () => { + stubCancel( + cancelReport({ OrdStatus: ScryptOrderStatus.NEW, ExecType: 'CancelRejected', CxlRejReason: 'TooLateToCancel' }), + ); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe( + ScryptCancellation.UNCONFIRMED, + ); + }); + + it.each([ + ['nothing filled', '0', ScryptCancellation.SETTLED], + ['a fill', '40', ScryptCancellation.EXECUTED], + ])('treats a rejected order with %s as terminal too', async (_label, cumQty, expected) => { + // a rejected order is as final as a cancelled one; a second opinion on what counts as terminal would + // be free to disagree with the first and leave such an order stuck + stubCancel(cancelReport({ OrdStatus: ScryptOrderStatus.REJECTED, CumQty: cumQty })); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe(expected); + }); + + it('settles nothing when a refused cancel reports a fill — the order is still open', async () => { + // a refusal carries the order's last known state, so a partially filled order that could NOT be + // cancelled reports a fill while remaining live. Reading the fill alone would call it finished and + // let the caller walk away from a reference that can still trade. + stubCancel( + cancelReport({ + OrdStatus: ScryptOrderStatus.PARTIALLY_FILLED, + ExecType: 'CancelRejected', + CxlRejReason: 'TooLateToCancel', + CumQty: '40', + LeavesQty: '60', + }), + ); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe( + ScryptCancellation.UNCONFIRMED, + ); + }); + + it('reports a fully filled order as executed', async () => { + stubCancel(cancelReport({ OrdStatus: ScryptOrderStatus.FILLED, CumQty: '100', LeavesQty: '0' })); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe(ScryptCancellation.EXECUTED); + }); + + it('settles nothing when the filled size cannot be read — that is not a zero', async () => { + stubCancel(cancelReport({ CumQty: undefined as unknown as string })); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe( + ScryptCancellation.UNCONFIRMED, + ); + }); + + it.each([ + ['', 'empty'], + [' ', 'whitespace'], + ])('settles nothing when the filled size is %p (%s) — that is missing, not zero', async (cumQty) => { + // Number('') is 0, not NaN, so this would otherwise pass a finite check and read as untouched + stubCancel(cancelReport({ CumQty: cumQty })); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe( + ScryptCancellation.UNCONFIRMED, + ); + }); + + it('settles nothing when the filled size is not a number at all', async () => { + stubCancel(cancelReport({ CumQty: 'abc' })); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe( + ScryptCancellation.UNCONFIRMED, + ); + }); + + it.each(['-1', '-0.5'])( + 'settles nothing when the filled size is %p — below zero is not untouched', + async (cumQty) => { + // A negative value is finite and parses cleanly, so it would pass every check above and then lose to + // `filled > 0` — reported as an order nothing ever traded. A cumulative filled size cannot be below + // zero, so this is a report not being understood, and misreading it grants exactly the false certainty + // that lets a live reference be walked away from. + stubCancel(cancelReport({ CumQty: cumQty })); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe( + ScryptCancellation.UNCONFIRMED, + ); + }, + ); + + it('waits past a PendingCancel report instead of taking it for the answer', async () => { + // the waiter resolves on its first match and then stops listening, so accepting the interim state + // would freeze it as the result and the real terminal report would never be seen. Exercises the + // matcher itself rather than mocking around it. + const connection = (service as any).connection; + let matcher: (reports: ScryptExecutionReport[]) => ScryptExecutionReport | null; + jest.spyOn(service as any, 'getTradePair').mockResolvedValue({ symbol: 'EUR/USDT' }); + jest.spyOn(connection, 'requestAndWaitForUpdate').mockImplementation(async (..._args: unknown[]) => { + matcher = _args[3] as typeof matcher; + return cancelReport(); + }); + + await service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT'); + + const pending = cancelReport({ OrdStatus: ScryptOrderStatus.PENDING_CANCEL }); + const terminal = cancelReport(); + expect(matcher([pending])).toBeNull(); + expect(matcher([pending, terminal])).toBe(terminal); + }); + + it('settles nothing when the cancel never came back', async () => { + stubCancel(new ScryptRequestTimeoutError('Timeout waiting for ExecutionReport update after 60000ms')); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe( + ScryptCancellation.UNCONFIRMED, + ); + }); + + it('files nothing when the cancel was refused — a refusal carries the last known state, not a verdict', async () => { + // for a reference the venue never had, that state reads as a live New order. Filing it would invent + // one, and the next lookup would call the order sent against a reference that never executed. + stubCancel( + cancelReport({ OrdStatus: ScryptOrderStatus.NEW, ExecType: 'CancelRejected', CxlRejReason: 'UnknownOrder' }), + ); + + await service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT'); + + expect((service as any).executionReports.has('dfx-lm-7')).toBe(false); + }); + + it('never files a cleanup cancellation under the order it cancelled', async () => { + // the confirmation carries the cancel request's id; filing it under the order would make a later + // status lookup read that order as terminally cancelled. A cleanup cancellation says nothing about + // the order as a whole — sibling references may still be unsettled and live — so a lookup reporting + // it as known would take it out of quarantine and let a replacement be opened beside them. + stubCancel(cancelReport({ CumQty: '40' })); + + await service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT'); + + expect((service as any).executionReports.has('dfx-lm-7')).toBe(false); + }); + }); + + describe('cancelIfOutstandingBySymbol', () => { + /** A complete report, so the fixture cannot drift from the contract cancelOrderBySymbol now promises. */ + function cancelReport(overrides: Partial = {}): ScryptExecutionReport { + return { + ClOrdID: 'cancel-req-2', + OrigClOrdID: 'legacy-ref-1', + Symbol: 'XRP/USDT', + Side: 'Sell', + OrdStatus: ScryptOrderStatus.CANCELED, + OrderQty: '10', + CumQty: '0', + LeavesQty: '0', + ...overrides, + }; + } + + it('cancels under the given symbol without deriving a trade pair', async () => { + const getTradePairSpy = jest.spyOn(service as any, 'getTradePair'); + const connection = (service as any).connection; + jest.spyOn(connection, 'requestAndWaitForUpdate').mockImplementation(async (..._args: unknown[]) => { + const [, [payload]] = _args as [unknown, [{ Symbol: string }]]; + expect(payload.Symbol).toBe('XRP/USDT'); + return cancelReport(); + }); + + await expect(service.cancelIfOutstandingBySymbol('legacy-ref-1', 'XRP/USDT')).resolves.toBe( + ScryptCancellation.SETTLED, + ); + expect(getTradePairSpy).not.toHaveBeenCalled(); + }); + + it('shares the same evaluation as cancelIfOutstanding — a fill is reported as executed here too', async () => { + const connection = (service as any).connection; + jest.spyOn(connection, 'requestAndWaitForUpdate').mockResolvedValue(cancelReport({ CumQty: '3' })); + + await expect(service.cancelIfOutstandingBySymbol('legacy-ref-1', 'XRP/USDT')).resolves.toBe( + ScryptCancellation.EXECUTED, + ); + }); + + it('goes unconfirmed and forgets the cached report the same way cancelIfOutstanding does', async () => { + const connection = (service as any).connection; + jest + .spyOn(connection, 'requestAndWaitForUpdate') + .mockRejectedValue(new ScryptRequestTimeoutError('Timeout waiting for ExecutionReport update after 60000ms')); + (service as any).executionReports.set('legacy-ref-1', { + ClOrdID: 'legacy-ref-1', + OrdStatus: ScryptOrderStatus.NEW, + }); + + await expect(service.cancelIfOutstandingBySymbol('legacy-ref-1', 'XRP/USDT')).resolves.toBe( + ScryptCancellation.UNCONFIRMED, + ); + expect((service as any).executionReports.has('legacy-ref-1')).toBe(false); + }); + + it('settles a refusal that says there is no such order — same UnknownOrder inference as cancelIfOutstanding', async () => { + const connection = (service as any).connection; + jest + .spyOn(connection, 'requestAndWaitForUpdate') + .mockResolvedValue( + cancelReport({ OrdStatus: ScryptOrderStatus.NEW, ExecType: 'CancelRejected', CxlRejReason: 'UnknownOrder' }), + ); + + await expect(service.cancelIfOutstandingBySymbol('legacy-ref-1', 'XRP/USDT')).resolves.toBe( + ScryptCancellation.SETTLED, + ); + }); + }); + describe('checkTrade — the amend write boundary', () => { function stubAmendPath(editOutcome: Error): void { jest.spyOn(service as any, 'getOrderStatus').mockResolvedValue({ @@ -816,7 +1330,16 @@ describe('ScryptService', () => { }); jest.spyOn(service as any, 'getTradePrice').mockResolvedValue(2); jest.spyOn(service as any, 'editOrder').mockRejectedValue(editOutcome); - jest.spyOn(service as any, 'cancelOrder').mockResolvedValue(undefined); + jest.spyOn(service as any, 'cancelOrder').mockResolvedValue({ + ClOrdID: 'cancel-req-1', + OrigClOrdID: 'dfx-lm-7', + Symbol: 'EUR/USDT', + Side: 'Sell', + OrdStatus: ScryptOrderStatus.CANCELED, + OrderQty: '5', + CumQty: '0', + LeavesQty: '0', + } satisfies ScryptExecutionReport); } it('propagates an unconfirmed amend instead of swallowing it', async () => { @@ -864,28 +1387,239 @@ describe('ScryptService', () => { expect((service as any).executionReports.has('dfx-lm-7')).toBe(true); }); - it('keeps waiting on a pending order however old it is — pending is observed, not unknown', async () => { - // quarantining it would make reconciliation find the reference, hand the order back, and the next - // completion check quarantine it again: a loop, not a resolution + it('keeps waiting on a pending order that is still young, without asking Scrypt to cancel it', async () => { + // without this, a fresh PENDING report would trigger a cancel-and-decide it has not earned yet — the + // bound exists precisely so a normal, seconds-long transition is never treated as stuck + jest.spyOn(service as any, 'getOrderStatus').mockResolvedValue({ + id: 'dfx-lm-7', + status: ScryptOrderStatus.PENDING_NEW, + remainingQuantity: 5, + }); + const cancelSpy = jest.spyOn(service, 'cancelIfOutstanding'); + + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', new Date())).resolves.toBe(false); + expect(cancelSpy).not.toHaveBeenCalled(); + }); + + it('keeps waiting when an old order has only just entered a pending state', async () => { + // Without the dwell-time clock, a price adjustment on an hours-old healthy order would cancel it + // because the old logic measured order age instead of time spent pending. + // + // Two passes on purpose: the FIRST pending poll only records the observation and returns, whatever + // the bound is measured against, so asserting it alone would pass just as happily against the age + // clock this replaced. The second pass is where the two differ — 120 minutes of age is past the + // bound, seconds of dwell time are not — so only this one actually pins the fix down. + jest.spyOn(service as any, 'getOrderStatus').mockResolvedValue({ + id: 'dfx-lm-7', + status: ScryptOrderStatus.PENDING_REPLACE, + remainingQuantity: 5, + }); + const cancelSpy = jest.spyOn(service, 'cancelIfOutstanding'); + const orderCreated = new Date(Date.now() - 120 * 60 * 1000); + + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', orderCreated)).resolves.toBe(false); + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', orderCreated)).resolves.toBe(false); + + expect(cancelSpy).not.toHaveBeenCalled(); + }); + + it('starts a fresh pending clock when a reference leaves pending before re-entering it', async () => { + // Without clearing pendingSince on the intervening non-pending status, a later PENDING entry would + // inherit the old, potentially expired wait instead of starting a new observation period. + jest + .spyOn(service as any, 'getOrderStatus') + .mockResolvedValueOnce({ + id: 'dfx-lm-7', + status: ScryptOrderStatus.PENDING_NEW, + remainingQuantity: 5, + }) + .mockResolvedValueOnce({ + id: 'dfx-lm-7', + status: ScryptOrderStatus.FILLED, + remainingQuantity: 0, + }) + .mockResolvedValueOnce({ + id: 'dfx-lm-7', + status: ScryptOrderStatus.PENDING_NEW, + remainingQuantity: 5, + }); + const cancelSpy = jest.spyOn(service, 'cancelIfOutstanding'); + + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', new Date())).resolves.toBe(false); + (service as any).pendingSince.set('dfx-lm-7', { + since: new Date(Date.now() - 6 * 60 * 1000), + lastSeen: new Date(Date.now() - 6 * 60 * 1000), + }); + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', new Date())).resolves.toBe(true); + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', new Date())).resolves.toBe(false); + + expect(cancelSpy).not.toHaveBeenCalled(); + }); + + it('keeps a reference that is still being observed, however long it has been pending', async () => { + // The prune sweep runs on lastSeen, never on since. A reference the venue has reported pending for + // more than a day is not stale — it is still polled every pass and retried under the cancel throttle. + // Pruning it by `since` would drop a live entry and hand it a fresh five-minute grace period, so the + // stuck order would never reach its bound at all. jest.spyOn(service as any, 'getOrderStatus').mockResolvedValue({ id: 'dfx-lm-7', status: ScryptOrderStatus.PENDING_NEW, remainingQuantity: 5, }); + const orderCreated = new Date(Date.now() - 30 * 60 * 60 * 1000); + + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', orderCreated)).resolves.toBe(false); + + // Age BOTH stamps, then observe the reference again. That second pass is the point: it takes the + // known-entry branch, which is the only place lastSeen is refreshed. Without asserting it here, that + // refresh could be deleted outright and the sweep below would still find the stamp the first pass + // wrote — the test would pass while the entry silently aged into being prunable. + const staleAt = new Date(Date.now() - 25 * 60 * 60 * 1000); + (service as any).pendingSince.set('dfx-lm-7', { since: staleAt, lastSeen: staleAt }); + // a day past its bound, so this pass reaches the cancel — the venue not confirming it is what keeps + // such a reference pending indefinitely, which is exactly the case the sweep must not collect + jest.spyOn(service, 'cancelIfOutstanding').mockResolvedValue(ScryptCancellation.UNCONFIRMED); + + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', orderCreated)).resolves.toBe(false); + expect((service as any).pendingSince.get('dfx-lm-7').lastSeen.getTime()).toBeGreaterThan(staleAt.getTime()); + // the clock itself must NOT be pushed forward, or the bound would never be reached + expect((service as any).pendingSince.get('dfx-lm-7').since).toEqual(staleAt); + + // a different reference entering pending for the first time is what triggers the sweep + jest.spyOn(service as any, 'getOrderStatus').mockResolvedValue({ + id: 'dfx-lm-8', + status: ScryptOrderStatus.PENDING_NEW, + remainingQuantity: 5, + }); + await expect(service.checkTrade('dfx-lm-8', 'EUR', 'USDT', orderCreated)).resolves.toBe(false); + + expect((service as any).pendingSince.has('dfx-lm-7')).toBe(true); + }); + + it('keeps waiting on a pending order past its bound when Scrypt will not confirm a cancel', async () => { + // the most important case here: without a confirmed cancel the order may still be live in the book, and + // giving it up on unconfirmed evidence could let the onFail chain place a second, genuinely competing buy; + // resolving false alone would not distinguish waiting after an unconfirmed cancel from never trying one + jest.spyOn(service as any, 'getOrderStatus').mockResolvedValue({ + id: 'dfx-lm-7', + status: ScryptOrderStatus.PENDING_NEW, + remainingQuantity: 5, + }); + const cancelSpy = jest.spyOn(service, 'cancelIfOutstanding').mockResolvedValue(ScryptCancellation.UNCONFIRMED); + + // PENDING_STUCK_AFTER_MINUTES is private, so backdate the service's observation clock directly instead + // of driving Date.now(); only the pending-dwell input needs to be past its bound in this test. + (service as any).pendingSince.set('dfx-lm-7', { + since: new Date(Date.now() - 6 * 60 * 1000), + lastSeen: new Date(Date.now() - 6 * 60 * 1000), + }); await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', new Date(Date.now() - 120 * 60 * 1000))).resolves.toBe( false, ); + expect(cancelSpy).toHaveBeenCalledTimes(1); }); - it('keeps waiting on a pending order that is still young', async () => { + it('fails a pending order past its bound once Scrypt confirms nothing can execute under it any more', async () => { + // without this, a trade stuck reporting PENDING forever would have no exit but a human noticing jest.spyOn(service as any, 'getOrderStatus').mockResolvedValue({ id: 'dfx-lm-7', status: ScryptOrderStatus.PENDING_NEW, remainingQuantity: 5, }); + jest.spyOn(service, 'cancelIfOutstanding').mockResolvedValue(ScryptCancellation.SETTLED); - await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', new Date())).resolves.toBe(false); + // PENDING_STUCK_AFTER_MINUTES is private, so backdate the service's observation clock directly instead + // of driving Date.now(); only the pending-dwell input needs to be past its bound in this test. + (service as any).pendingSince.set('dfx-lm-7', { + since: new Date(Date.now() - 6 * 60 * 1000), + lastSeen: new Date(Date.now() - 6 * 60 * 1000), + }); + + await expect( + service.checkTrade('dfx-lm-7', 'EUR', 'USDT', new Date(Date.now() - 6 * 60 * 1000)), + ).rejects.toBeInstanceOf(ScryptOrderStuckPendingError); + expect((service as any).pendingSince.has('dfx-lm-7')).toBe(false); + }); + + it('completes a pending order past its bound when Scrypt confirms it filled after all', async () => { + // without this, a trade that actually filled would be failed instead of counted as complete + jest.spyOn(service as any, 'getOrderStatus').mockResolvedValue({ + id: 'dfx-lm-7', + status: ScryptOrderStatus.PENDING_NEW, + remainingQuantity: 5, + }); + jest.spyOn(service, 'cancelIfOutstanding').mockResolvedValue(ScryptCancellation.EXECUTED); + + // PENDING_STUCK_AFTER_MINUTES is private, so backdate the service's observation clock directly instead + // of driving Date.now(); only the pending-dwell input needs to be past its bound in this test. + (service as any).pendingSince.set('dfx-lm-7', { + since: new Date(Date.now() - 6 * 60 * 1000), + lastSeen: new Date(Date.now() - 6 * 60 * 1000), + }); + + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', new Date(Date.now() - 6 * 60 * 1000))).resolves.toBe( + true, + ); + expect((service as any).pendingSince.has('dfx-lm-7')).toBe(false); + }); + + it('does not retry the cancel write on the very next pass after an unconfirmed attempt', async () => { + // the cancel is a WRITE and checkRunningOrders runs nominally every 10 seconds, and possibly more often + // within one pass — without a retry floor per reference this would fire a fresh cancel on every one of + // those calls for as long as the venue keeps not confirming it + jest.spyOn(service as any, 'getOrderStatus').mockResolvedValue({ + id: 'dfx-lm-7', + status: ScryptOrderStatus.PENDING_NEW, + remainingQuantity: 5, + }); + const cancelSpy = jest.spyOn(service, 'cancelIfOutstanding').mockResolvedValue(ScryptCancellation.UNCONFIRMED); + const orderCreated = new Date(Date.now() - 6 * 60 * 1000); + + // PENDING_STUCK_AFTER_MINUTES is private, so backdate the service's observation clock directly instead + // of driving Date.now(); only the pending-dwell input needs to be past its bound in this test. + (service as any).pendingSince.set('dfx-lm-7', { + since: new Date(Date.now() - 6 * 60 * 1000), + lastSeen: new Date(Date.now() - 6 * 60 * 1000), + }); + + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', orderCreated)).resolves.toBe(false); + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', orderCreated)).resolves.toBe(false); + + expect(cancelSpy).toHaveBeenCalledTimes(1); + }); + + it('retries the cancel write again once the retry floor has passed', async () => { + // Mirrors the previous test but advances past PENDING_CANCEL_RETRY_MINUTES between passes: without + // this, an implementation whose retry floor never expires would look identical to a correct one to + // the test above alone, and a permanently blocked cancel is itself a new stuck-forever wait state. + jest.spyOn(service as any, 'getOrderStatus').mockResolvedValue({ + id: 'dfx-lm-7', + status: ScryptOrderStatus.PENDING_NEW, + remainingQuantity: 5, + }); + const cancelSpy = jest.spyOn(service, 'cancelIfOutstanding').mockResolvedValue(ScryptCancellation.UNCONFIRMED); + const orderCreated = new Date(Date.now() - 6 * 60 * 1000); + + // PENDING_STUCK_AFTER_MINUTES is private, so backdate the service's observation clock directly instead + // of driving Date.now(); only the pending-dwell input needs to be past its bound in this test. + (service as any).pendingSince.set('dfx-lm-7', { + since: new Date(Date.now() - 6 * 60 * 1000), + lastSeen: new Date(Date.now() - 6 * 60 * 1000), + }); + + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', orderCreated)).resolves.toBe(false); + expect(cancelSpy).toHaveBeenCalledTimes(1); + + // PENDING_CANCEL_RETRY_MINUTES is a private module-level constant (1 minute) with no export to + // import in this test, so the recorded attempt is backdated directly on the service's private map + // instead of driving Date.now() itself — this is the one throttle input the test needs to move, and + // moving only it keeps the assertion tied to the retry floor, not incidentally to the pending dwell. + const pastRetryFloor = new Date(Date.now() - 2 * 60 * 1000); + (service as any).pendingCancelAttempts.set('dfx-lm-7', pastRetryFloor); + + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', orderCreated)).resolves.toBe(false); + expect(cancelSpy).toHaveBeenCalledTimes(2); }); it('cancels on an explicit rejection, but reports the refusal and the spent reference', async () => { diff --git a/src/integration/exchange/services/scrypt-websocket-connection.ts b/src/integration/exchange/services/scrypt-websocket-connection.ts index 089a7527a3..e1da99de52 100644 --- a/src/integration/exchange/services/scrypt-websocket-connection.ts +++ b/src/integration/exchange/services/scrypt-websocket-connection.ts @@ -113,6 +113,31 @@ export class ScryptUnconfirmedWriteError extends Error { */ export class ScryptOrderNotFoundError extends Error {} +/** + * A trade whose venue reference this process has continuously observed reporting PENDING_NEW/PENDING_CANCEL/ + * PENDING_REPLACE for longer than PENDING_STUCK_AFTER_MINUTES, and whose outstanding reference the venue then + * answered on an explicit cancel request with `ScryptCancellation.SETTLED` — one of two distinct qualities of + * answer, per that enum's own documentation: a terminal cancel with nothing filled, or the venue not + * recognising the reference at all (`SCRYPT_UNKNOWN_ORDER`), which is an inference from its own words rather + * than a statement about execution. + * + * The bound measures how long this process has continuously observed the reference as PENDING, beginning + * with its first such observation. It does not measure the age of the order itself. + * + * Both qualities of SETTLED carry the weight this error rests on, but not for the same reason. A terminal + * cancel states it outright. `UnknownOrder` rests on the inference documented at SCRYPT_UNKNOWN_ORDER: a + * venue that does not recognise a reference cannot execute anything under it. Deliberately NOT argued as + * the venue contradicting its own last answer — the status read here comes from the cached execution report + * and is not re-fetched before the cancel, so a live push may have moved it on in between, and "its last + * answer" is then not what this branch saw. + * + * Distinct from {@link ScryptOrderNotFoundError}: that one means the venue cannot find the order at all, an + * unresolved blind spot. Here the venue found it, answered PENDING, and then settled it on request — which is + * why the caller may fail the order instead of quarantining it. No change in behaviour from this wording, + * only a more honest account of what SETTLED actually rests on. + */ +export class ScryptOrderStuckPendingError extends Error {} + /** * An amend the venue refused. The replacement was never created, so the ORIGINAL order is still live — and * its reference is spent, because the venue requires references to be unique. Carries it so the caller can diff --git a/src/integration/exchange/services/scrypt.service.ts b/src/integration/exchange/services/scrypt.service.ts index 9c57bd4049..ee07e4c992 100644 --- a/src/integration/exchange/services/scrypt.service.ts +++ b/src/integration/exchange/services/scrypt.service.ts @@ -9,6 +9,7 @@ import { PricingProvider } from 'src/subdomains/supporting/pricing/services/inte import { ScryptBalance, ScryptBalanceTransaction, + ScryptCancellation, ScryptDepositStatus, ScryptExecutionReport, ScryptMarketDataSnapshot, @@ -25,6 +26,7 @@ import { ScryptTransactionType, ScryptWithdrawResponse, ScryptWithdrawStatus, + isTerminalScryptOrderStatus, } from '../dto/scrypt.dto'; import { TradeChangedException } from '../exceptions/trade-changed.exception'; import { @@ -33,6 +35,7 @@ import { ScryptAmendRejectedError, ScryptMessageType, ScryptOrderNotFoundError, + ScryptOrderStuckPendingError, ScryptUnconfirmedWriteError, ScryptVenueRejectionError, ScryptWebSocketConnection, @@ -40,9 +43,72 @@ import { /** * After this long without a usable answer, an order the venue once acknowledged is treated as lost rather - * than merely slow. Shared by the "cannot be found" and the "stuck pending" paths so both give up together. + * than merely slow. One use only: the branch below where the status lookup returns nothing at all. The + * pending states deliberately do NOT consult it — a venue that still reports PENDING_NEW is answering, and + * this constant is about silence. They are bounded separately by PENDING_STUCK_AFTER_MINUTES, which asks + * for a cancel rather than declaring the order lost, because there the reference is known to exist. + * + * Kept equal to SCRYPT_UNOBSERVABLE_QUARANTINE_MINUTES in the adapter, which documents itself as matching + * this value — the two are the pair of routes out of a silent order and must not drift apart. Five rather + * than sixty so that quarantine plus the abandon bound stay inside the ten-minute ceiling. Safe at this + * length because it only fires while the venue does not know the reference at all: an order it is still + * working is returned by the status lookup and never reaches here, whatever its age. Measured over 60 days, + * Scrypt trades complete in a median of 0.2 and a maximum of 1.0 minutes. */ -const ORDER_LOST_AFTER_MINUTES = 60; +const ORDER_LOST_AFTER_MINUTES = 5; + +/** + * PENDING_NEW / PENDING_CANCEL / PENDING_REPLACE are meant to be a transition lasting seconds — the venue + * is in the middle of accepting, cancelling or replacing the order. Measured over 60 days, Scrypt trades + * spend a median of 0.2 and a maximum of 1.0 minutes reaching a terminal or open state, so a reference that + * has continuously answered PENDING for more than five minutes is not merely slow, it is stuck. + * + * Measured from when THIS process first observes the venue continuously reporting a PENDING state for the + * reference, not from the order's creation. A process restart starts that observation clock afresh; this can + * only extend the bound, never cause the adapter to give up too early. + * + * Deliberately its own constant rather than reusing ORDER_LOST_AFTER_MINUTES, even though both happen to be + * five: that one bounds SILENCE from the venue (no status at all), this one bounds an ANSWER that makes no + * progress. Merging them would let a later change meant for one quietly shift the other along with it. + * + * The bound never ends the order by itself — the exit is canceling and restarting, never a bare give-up: a + * trade that might still be sitting in the book must not be abandoned unconfirmed, or the onFail chain could + * place a second, genuinely competing buy alongside it. + */ +const PENDING_STUCK_AFTER_MINUTES = 5; + +/** + * Minimum wait between two cancel attempts for the SAME pending reference, once it is past its bound. + * + * The cancel below is a WRITE to the venue, and `checkRunningOrders` is driven by a cron that runs nominally + * every ten seconds, and possibly more often within one pass — the cron itself has a lock, a jitter lead-in + * and a process-disable gate, and `processPipelines`'s own `while (hasChanges)` loop can invoke it again + * before the next tick. Without a floor here, an UNCONFIRMED answer would draw a fresh cancel on every one of + * those calls for as long as the venue keeps not confirming it. One minute, matching the cooldown floor the + * analogous quarantine-cancel throttle uses (`UNCERTAIN_RESOLVE_MIN_INTERVAL_MS` in + * liquidity-management-pipeline.service.ts), for the same stated reason: "a cancellation the venue will not + * confirm must not retry on every ten-second tick." This only slows the write down — it does not move + * PENDING_STUCK_AFTER_MINUTES itself, so the order becomes eligible for a cancel after exactly the same + * pending dwell time either way; only how often an unconfirmed attempt may repeat changes. + */ +const PENDING_CANCEL_RETRY_MINUTES = 1; + +// The venue answers a refused cancel with an execution report rather than a separate reject message, so the +// refusal has to be read off these two fields. `UnknownOrder` is the one reason treated as settling +// anything. +// +// That reading is an inference, not a documented guarantee — the protocol spec lists the reason without +// defining it, so "never existed" cannot be distinguished from "not processed yet" from the value alone. +// What it rests on: the caller only cancels an order after either a failed status lookup has outlived the +// full ORDER_LOST_AFTER_MINUTES window in which its request could still be in flight, or a successful lookup +// has continuously reported PENDING for the full PENDING_STUCK_AFTER_MINUTES window. The latter premise is +// even stronger: the venue just confirmed that the reference existed, then no longer knew it at the cancel — +// a stronger signal than silence alone. Note what that does and does not cover — the lookup stops at the +// first reference the venue does not show, so for every other reference of the same order this refusal is the +// only negative answer there is. Age plus one refusal is the strongest evidence this protocol offers. Every +// other reason (too late, rate limited, already pending) settles nothing and is waited out. +const SCRYPT_CANCEL_REJECTED = 'CancelRejected'; +const SCRYPT_UNKNOWN_ORDER = 'UnknownOrder'; // The bulk streams a reconnect catch-up restores; the live subscriptions cover everything else. type CatchUpStream = ScryptMessageType.EXECUTION_REPORT | ScryptMessageType.BALANCE_TRANSACTION; @@ -57,6 +123,13 @@ export class ScryptService extends PricingProvider { private readonly balances?: AsyncSubscription>; private readonly executionReports: Map = new Map(); private readonly balanceTransactions: Map = new Map(); + // Throttle for the WRITE in the PENDING branch of checkTrade — see PENDING_CANCEL_RETRY_MINUTES. Keyed by + // clOrdId, value is the end of the last cancel attempt for that reference. + private readonly pendingCancelAttempts: Map = new Map(); + // Tracks how long THIS process has continuously seen a reference report a PENDING status — see + // PENDING_STUCK_AFTER_MINUTES. `since` measures that bound; `lastSeen` is only for 24-hour cleanup. + // Cleared once the reference leaves the pending states, so a later re-entry starts fresh. + private readonly pendingSince: Map = new Map(); private catchUpInProgress = false; private catchUpPending = false; private lastCatchUpAt?: number; @@ -162,9 +235,23 @@ export class ScryptService extends PricingProvider { // A warm-up that loaded BOTH streams is exactly what a catch-up round does, so it claims the first slot and a // reconnect right after boot waits it out instead of repeating it. If either leg failed the caches are not - // whole, and the next reconnect must repair immediately rather than sit out the interval on stale state. + // whole: without an immediate retry the only refill path is onReconnect, so a stable socket after a failed + // warm-up would leave balanceTransactions / executionReports empty forever. An empty cache no longer blocks + // confirmWithdrawalAbsent (absence is decided from the fresh venue reply + live recheck only), but findWithdrawal + // and that live recheck still benefit from a filled cache, so incomplete boot warm-up must still be retried + // promptly rather than waiting for a human or a later reconnect. Reuse catchUpAfterReconnect (both streams, + // existing pacing / retry) rather than inventing a second timer type; trigger is only a true Promise rejection + // on a leg, never "empty array" (that is a successful warm-up with no rows). void Promise.all([executionWarmUp, balanceWarmUp]).then(([executionLoaded, balanceLoaded]) => { - if (executionLoaded && balanceLoaded) this.lastCatchUpAt = Date.now(); + if (executionLoaded && balanceLoaded) { + this.lastCatchUpAt = Date.now(); + return; + } + + this.logger.warn( + `Scrypt boot warm-up incomplete (executionLoaded=${executionLoaded}, balanceLoaded=${balanceLoaded}) — triggering catch-up without waiting for a reconnect`, + ); + void this.catchUpAfterReconnect().catch((e) => this.logger.error('Scrypt catch-up retry failed:', e)); }); this.connection.onReconnect(() => this.catchUpAfterReconnect()); @@ -187,7 +274,7 @@ export class ScryptService extends PricingProvider { } private isTerminalExecutionReport(r: ScryptExecutionReport): boolean { - return [ScryptOrderStatus.FILLED, ScryptOrderStatus.CANCELED, ScryptOrderStatus.REJECTED].includes(r.OrdStatus); + return isTerminalScryptOrderStatus(r.OrdStatus); } /** @@ -215,7 +302,17 @@ export class ScryptService extends PricingProvider { // Bulk (age-bounded) warm-up/catch-up path only — live subscriptions must cache directly via cacheExecutionReport/cacheBalanceTransaction, see constructor. private applyBalanceTransactions(transactions: ScryptBalanceTransaction[]): void { const cacheMaxAge = Util.daysBefore(365); - for (const t of transactions) if (new Date(t.Timestamp) >= cacheMaxAge) this.cacheBalanceTransaction(t); + for (const t of transactions) { + // Field priority matches the rest of this file: Timestamp first, TransactTime only when Timestamp is missing. + // Missing or unreadable stamp → cache conservatively (never drop). The bulk age filter must not discard a + // withdrawal we later need for findWithdrawal or the live recheck in confirmWithdrawalAbsent — a dropped + // row is a payout we cannot rediscover. This is a deliberate, documented fallback (cache on doubt), not a + // silent default. + const raw = t.Timestamp ?? t.TransactTime; + if (!raw || Number.isNaN(new Date(raw).getTime()) || new Date(raw) >= cacheMaxAge) { + this.cacheBalanceTransaction(t); + } + } } // After a WS reconnect, re-fetch balance transactions + execution reports so an event missed during the outage @@ -585,6 +682,75 @@ export class ScryptService extends PricingProvider { return found; } + /** + * Confirm that the venue's transaction history has no record of this withdrawal reference. + * + * Scrypt has no cancel/storno for withdrawals. A quarantined withdrawal therefore cannot be cleared the way + * a trade is. The automatic exit is confirmed absence: a fresh bulk fetch was returned and this `clReqId` + * is not in it — whether that history has rows or not. The caller abandons on that basis rather than + * claiming a not-sent release. + * + * No local consistency gate / cache-anchor check. Scrypt withdrawal destinations are exclusively DFX-owned + * addresses ("Auszahlungsadressen bei Scrypt gehören alle ausnahmslos der DFX AG"). A second payout would + * move funds only between DFX accounts — an internal rebooking, not a loss and not a compliance incident + * ("eine doppelte Auszahlung wäre absolut akzeptabel"). The former gates traded that accepted non-risk for a + * forbidden permanent wait: they required cache anchors that can be structurally absent (no row with + * ClReqID, no row older than the order, 365-day cache bound), so confirmWithdrawalAbsent returned false + * forever and the order waited on a human to refill the cache. "Die Kombination aus Scrypt und Warten auf + * einen Menschen ist NICHT ERLAUBT." + * + * Incomplete or truncated venue replies (pagination cut-off, partial answer) can now cause a second + * withdrawal. That is the deliberate trade-off accepted by the orderer — not an overlooked gap. + * + * Remaining `false` branches react only to a fetch failure or a live-cache race hit (the reference appeared + * in the live map while the bulk fetch was in flight). An empty history is no longer one of them: on a + * fresh or long-dormant Scrypt account the trade history genuinely has no rows, and a successful `[]` is + * treated exactly like any other successful reply that does not name the reference — not as a reason to + * wait. It is not evidence the history was complete: a truncated reply can arrive this way too, which is + * the trade-off stated above, accepted rather than overlooked. That is still not the same case as a real + * outage: an outage does not come back as an empty array, it throws, and lands in the catch above. The two + * look alike only on paper — a thrown error and a successful empty reply are deliberately handled + * differently. + * + * @returns true when the live cache does not hold `clReqId` after the fetch and the fresh history (empty or + * not) does not contain it. false on fetch failure, live-race hit, or when the reference is present in the + * fresh reply. + */ + async confirmWithdrawalAbsent(clReqId: string): Promise { + let fresh: ScryptBalanceTransaction[]; + try { + fresh = await this.connection.fetchAll(ScryptMessageType.BALANCE_TRANSACTION); + } catch (e) { + this.logger.warn(`confirmWithdrawalAbsent(${clReqId}): could not fetch full transaction history: ${e.message}`); + return false; + } + + // An empty reply is a successful answer that does not name the reference — not a reason to wait. See + // the JSDoc above for why this is no longer a `false` branch: it carries exactly the weight of a + // non-empty history without `clReqId` below, no more, and is not proof the history was complete. + if (!fresh.length) { + this.logger.info( + `confirmWithdrawalAbsent(${clReqId}): venue returned an empty transaction history — the reference cannot exist in a history with no rows at all`, + ); + } + + const freshIds = new Set(fresh.map((t) => t.ClReqID).filter((id): id is string => Boolean(id))); + + // A live subscription (or catch-up) can write clReqId into this.balanceTransactions while the bulk fetch + // is still open — freshIds then misses it, and returning true would let the caller abandon and replan a + // second withdrawal. Only a re-read of the LIVE map after the await sees that race; if the id is there, + // absence is not confirmed (false, not a hard failure). This is the one remaining positive observation + // that can still block absence confirmation without requiring a local cache anchor. + if (this.balanceTransactions.has(clReqId)) { + this.logger.warn( + `confirmWithdrawalAbsent(${clReqId}): reference appeared in the live cache while the history fetch was in flight — cannot conclude absence`, + ); + return false; + } + + return !freshIds.has(clReqId); + } + /** * @param since lower bound for the fallback history fetch. A caller that knows when its reference can * earliest have existed passes it here, so a lookup for an absent order does not pull a full 30 days of @@ -624,6 +790,17 @@ export class ScryptService extends PricingProvider { }; } + // Prunes on every write rather than on a timer or a terminal event: this throttle only ever needs the MOST + // RECENT attempt, so nothing is lost by dropping an old one, and 24 hours is far beyond + // PENDING_STUCK_AFTER_MINUTES (single-digit minutes) — an entry that old belongs to a reference that has + // long since resolved or moved on, not one still cycling through the PENDING branch below. + private recordPendingCancelAttempt(clOrdId: string): void { + this.pendingCancelAttempts.set(clOrdId, new Date()); + + const dayAgo = Util.hoursBefore(24); + for (const [id, at] of this.pendingCancelAttempts) if (at < dayAgo) this.pendingCancelAttempts.delete(id); + } + /** * @param replacementClOrdId reference to use if this check has to amend or restart the order. Must be * reproducible from the order row by the caller, so a timed-out replacement stays findable. @@ -632,7 +809,7 @@ export class ScryptService extends PricingProvider { clOrdId: string, from: string, to: string, - orderCreated?: Date, + orderCreated: Date, replacementClOrdId?: string, // Invoked immediately before a replacement is sent, so the caller can make the reference durable first. // Without that, a replacement whose confirmation is lost is neither the current reference nor a spent @@ -641,8 +818,8 @@ export class ScryptService extends PricingProvider { ): Promise { const orderInfo = await this.getOrderStatus(clOrdId); if (!orderInfo) { - // If the order is older than 1 hour and still not found, it's lost - const ageMinutes = orderCreated ? Util.minutesDiff(orderCreated) : 0; + // Past its bound and still not found anywhere: treat it as lost rather than keep polling for it. + const ageMinutes = Util.minutesDiff(orderCreated); if (ageMinutes > ORDER_LOST_AFTER_MINUTES) { throw new ScryptOrderNotFoundError( `Order ${clOrdId} not found after ${Math.round(ageMinutes)} minutes — it may have completed or been cancelled outside of tracked state`, @@ -653,6 +830,13 @@ export class ScryptService extends PricingProvider { return false; } + if ( + orderInfo.status !== ScryptOrderStatus.PENDING_NEW && + orderInfo.status !== ScryptOrderStatus.PENDING_CANCEL && + orderInfo.status !== ScryptOrderStatus.PENDING_REPLACE + ) + this.pendingSince.delete(clOrdId); + switch (orderInfo.status) { case ScryptOrderStatus.NEW: case ScryptOrderStatus.PARTIALLY_FILLED: { @@ -692,7 +876,8 @@ export class ScryptService extends PricingProvider { this.logger.verbose(`Could not update order ${clOrdId}, attempting cancel: ${e.message}`); let cancelConfirmed = true; try { - await this.cancelOrder(clOrdId, from, to); + const cancelReport = await this.cancelOrder(clOrdId, from, to); + cancelConfirmed = cancelReport.OrdStatus === ScryptOrderStatus.CANCELED; } catch (cancelError) { // The cancel is a write too. Unconfirmed, it may well have taken effect at the venue while the // cached report still shows the order open — and a non-terminal entry is never refreshed, so @@ -771,14 +956,81 @@ export class ScryptService extends PricingProvider { case ScryptOrderStatus.PENDING_NEW: case ScryptOrderStatus.PENDING_CANCEL: - case ScryptOrderStatus.PENDING_REPLACE: - // Deliberately just waits, however old the order is. A pending report is an OBSERVATION — we know - // where the order stands — so it is not an unknown outcome and must not be quarantined: reconciliation - // would find the reference, hand the order straight back, and the next completion check would - // quarantine it again. An order that stays pending too long is a stuck order, which the monitoring - // counter surfaces; it is not an unresolved one. - this.logger.verbose(`Order ${clOrdId} is pending (${orderInfo.status}), waiting...`); + case ScryptOrderStatus.PENDING_REPLACE: { + // A pending report is an OBSERVATION — we know where the order stands — so on its own it is not an + // unknown outcome and must not be quarantined: reconciliation would find the reference, hand the + // order straight back, and the next completion check would quarantine it again. But "observed" is not + // the same claim as "running": PENDING_* is meant to last seconds (see PENDING_STUCK_AFTER_MINUTES), + // so past that bound this stops merely waiting and asks the venue to settle the question. The clock + // alone is never the exit, though — only a confirmed cancel is: a trade that might still be sitting in + // the book must not be given up on unconfirmed evidence, or the onFail chain could place a second, + // genuinely competing buy right next to it. That cancel is a WRITE to the venue, so it is throttled to + // at most one attempt per PENDING_CANCEL_RETRY_MINUTES per reference — otherwise an UNCONFIRMED answer + // would draw a fresh cancel on every checkRunningOrders call, which runs nominally every ten seconds, + // and possibly more often within one pass. + const pendingEntry = this.pendingSince.get(clOrdId); + if (!pendingEntry) { + const now = new Date(); + this.pendingSince.set(clOrdId, { since: now, lastSeen: now }); + + // Clean up by `lastSeen`, not `since`: a reference may remain continuously pending indefinitely while + // still being observed and retried by the PENDING_CANCEL_RETRY_MINUTES throttle below. Cleaning by + // `since` would drop an active reference and incorrectly grant a new grace period on its next tick. + const dayAgo = Util.hoursBefore(24); + for (const [id, entry] of this.pendingSince) if (entry.lastSeen < dayAgo) this.pendingSince.delete(id); + + this.logger.verbose(`Order ${clOrdId} is pending (${orderInfo.status}), waiting...`); + return false; + } + + pendingEntry.lastSeen = new Date(); + const pendingMinutes = Util.minutesDiff(pendingEntry.since); + if (pendingMinutes <= PENDING_STUCK_AFTER_MINUTES) { + this.logger.verbose(`Order ${clOrdId} is pending (${orderInfo.status}), waiting...`); + return false; + } + + const lastCancelAttempt = this.pendingCancelAttempts.get(clOrdId); + if (lastCancelAttempt && Util.minutesDiff(lastCancelAttempt) < PENDING_CANCEL_RETRY_MINUTES) { + this.logger.verbose( + `Order ${clOrdId} is pending (${orderInfo.status}) past its bound, but its last cancel attempt was ` + + `less than ${PENDING_CANCEL_RETRY_MINUTES} minute(s) ago — waiting before retrying the write`, + ); + return false; + } + + let cancellation: ScryptCancellation; + try { + cancellation = await this.cancelIfOutstanding(clOrdId, from, to); + } finally { + this.recordPendingCancelAttempt(clOrdId); + } + + if (cancellation === ScryptCancellation.EXECUTED) { + this.pendingCancelAttempts.delete(clOrdId); + this.pendingSince.delete(clOrdId); + this.logger.verbose(`Order ${clOrdId} was pending past its bound, but Scrypt confirms it filled`); + return true; + } + + if (cancellation === ScryptCancellation.SETTLED) { + this.pendingCancelAttempts.delete(clOrdId); + this.pendingSince.delete(clOrdId); + const ageMinutes = Util.minutesDiff(orderCreated); + throw new ScryptOrderStuckPendingError( + `Order ${clOrdId} is ${Math.round(ageMinutes)} minutes old and currently reports status ` + + `${orderInfo.status}, past the ${PENDING_STUCK_AFTER_MINUTES}-minute pending bound, and Scrypt ` + + `confirms nothing can execute under it any more`, + ); + } + + // UNCONFIRMED: nothing may be concluded. Without a confirmed cancel the reference may still be live + // in the book, so the order keeps waiting rather than being given up unconfirmed. + this.logger.warn( + `Order ${clOrdId} is pending (${orderInfo.status}) past its bound, but Scrypt would not confirm a cancel, waiting...`, + ); return false; + } } } @@ -845,8 +1097,154 @@ export class ScryptService extends PricingProvider { }; } - private async cancelOrder(clOrdId: string, from: string, to: string): Promise { + /** + * Ask the venue to make sure a reference cannot execute any more, and report what that established. + * + * For giving up on an order whose outcome was never observed. The danger there is never the order itself + * but a request still live in the book: hand the funds back to a rule while one sits open and a late fill + * spends them twice. Cancelling removes that possibility outright, which beats estimating when it has + * passed — and unlike a re-send, a cancel can never create anything. + * + * Three outcomes, because a cancel does not only ever mean "nothing happened": + * - SETTLED — terminal with nothing filled (cancelled or rejected), or the venue does not know the + * reference at all. Both mean nothing can execute under it, which is what lets the caller give the + * order up — the first outright, the second as an inference from the venue's own words rather than a + * statement about execution. See SCRYPT_UNKNOWN_ORDER for what that inference rests on. + * - EXECUTED — it reached a terminal state with something filled. Like a cancelled reference it cannot + * trade further, so the caller may give the order up; the fill has already moved the venue balance + * that the rule replans from. Reported separately from SETTLED because "something happened here" is + * worth seeing in a log and worth reconciling against. + * + * Terminal is the operative word: a refused cancel carries the order's last known state, so a + * partially filled order that could NOT be cancelled reports a fill while staying wide open. + * - UNCONFIRMED — no usable answer. Nothing may be concluded from it. + * + * Symbol resolution is the one thing that differs between this and {@link cancelIfOutstandingBySymbol}; + * everything else (send, evaluate, guard, catch) lives once in {@link cancelIfOutstandingCore} so neither + * can drift from the other. + */ + async cancelIfOutstanding(clOrdId: string, from: string, to: string): Promise { + return this.cancelIfOutstandingCore(clOrdId, async () => (await this.getTradePair(from, to)).symbol); + } + + /** + * Same outcome as {@link cancelIfOutstanding}, for a reference whose trade pair cannot be rebuilt locally — + * typically a command no longer in `ScryptAdapterCommands` (rename/removal), where a `paramMap` that + * happens to still carry a `tradeAsset` is not a guarantee the command was ever a plain sell/buy. + * + * The venue's own order-status reply already names the symbol a reference lives under + * (`ScryptOrderInfo.symbol`), straight from the venue rather than reconstructed from configuration that may + * no longer match reality (a delisted or renamed security would make `getTradePair` throw even though the + * order in question is still very much live under its original symbol). Every reference the venue can + * still show us is therefore cancellable through the symbol it hands back — there is no "symbol not + * determinable" wait path left, only the venue not knowing the reference at all (SETTLED, same inference as + * the `UnknownOrder` case below) or not answering at all (the caller's own lookup decides what to do with + * that, not this method). + */ + async cancelIfOutstandingBySymbol(clOrdId: string, symbol: string): Promise { + return this.cancelIfOutstandingCore(clOrdId, async () => symbol); + } + + private async cancelIfOutstandingCore( + clOrdId: string, + // Deferred rather than a plain string: the symbol lookup used by cancelIfOutstanding has to run INSIDE + // this method's try, exactly as it did when cancelOrder resolved it internally — otherwise a failing + // getTradePair would escape uncaught instead of settling into UNCONFIRMED like every other cancel failure. + resolveSymbol: () => Promise, + ): Promise { + try { + const symbol = await resolveSymbol(); + const report = await this.cancelOrderBySymbol(clOrdId, symbol); + + const filled = Number(report.CumQty); + + // An unreadable quantity is not a zero one. Concluding "nothing filled" from a value that could not + // be parsed is exactly how a real fill gets dropped, so it settles nothing and the caller waits. + // + // Emptiness has to be caught separately: Number('') and Number(' ') are 0, not NaN, so a missing + // quantity would otherwise pass the finite check and read as an untouched order. A negative one is + // rejected for the same reason rather than compared away: a cumulative filled size cannot be below + // zero, so a venue reporting one is not describing an untouched order — it is not being understood, + // and only the checks below would quietly treat it as though nothing had traded. + if (!report.CumQty?.trim() || !Number.isFinite(filled) || filled < 0) { + this.logger.warn(`Cancel of order ${clOrdId} reported an unreadable filled size (${report.CumQty})`); + + return ScryptCancellation.UNCONFIRMED; + } + + const refusedAsUnknown = + report.ExecType === SCRYPT_CANCEL_REJECTED && report.CxlRejReason === SCRYPT_UNKNOWN_ORDER; + + // Checked before anything else: a report claiming the venue has no record of this order while + // reporting a fill on it disagrees with itself, and that is true whatever its status says. Deciding + // on the status first would let the same contradiction through with a terminal one attached. + if (refusedAsUnknown && filled > 0) { + this.logger.warn( + `Cancel of order ${clOrdId} was refused as unknown yet reports ${report.CumQty} filled — the report contradicts itself, settling nothing`, + ); + + return ScryptCancellation.UNCONFIRMED; + } + + // Only a terminal state answers the question this method asks. A refused cancel comes back carrying + // the order's LAST KNOWN state, so a partially filled order that could not be cancelled reports a + // fill while remaining wide open — reading the fill alone would call that finished and let the + // caller walk away from a reference that can still trade. + // + // Which states are terminal is decided in one place for this venue, not restated here: a rejected + // order is just as final as a cancelled one, and a second list would be free to disagree with the + // first — leaving an order that provably cannot trade stuck for want of being recognised. + if (this.isTerminalExecutionReport(report)) { + if (filled > 0) { + this.logger.warn( + `Cancel of order ${clOrdId} came back terminal with ${report.CumQty} already filled — it executed, and the fill has to be reconciled against the venue balance`, + ); + + return ScryptCancellation.EXECUTED; + } + + return ScryptCancellation.SETTLED; + } + + // The venue does not know this reference. Taken together with the order's age and its failed status + // lookup, that is treated as settled — see SCRYPT_UNKNOWN_ORDER for what that evidence covers and + // why it is an inference rather than a guarantee. + if (refusedAsUnknown) { + this.logger.verbose(`Scrypt has no such order to cancel for ${clOrdId}`); + + return ScryptCancellation.SETTLED; + } + + this.logger.warn( + `Cancel of order ${clOrdId} left it in state ${report.OrdStatus}${ + report.CxlRejReason ? ` (${report.CxlRejReason})` : '' + } — nothing settled`, + ); + + return ScryptCancellation.UNCONFIRMED; + } catch (e) { + // No rejection branch here on purpose: this venue answers a refused cancel with an execution report, + // not an exception, and that is read above. What reaches this catch is anything that stopped the cancel + // from being answered — the symbol lookup it starts with, or the send and its wait. + // + // Not all of those got as far as writing, but this cannot tell which did, and that is the whole reason + // to treat them alike: an unconfirmed cancel may have taken effect at the venue while the cached report + // still shows the order open, and a non-terminal entry is never refreshed, so every later check would + // wait on a picture that cannot change. Dropping it costs one lookup when nothing was ever sent, and + // avoids a permanently stale one when something was. + this.forgetExecutionReport(clOrdId); + this.logger.warn(`Cancel of order ${clOrdId} went unconfirmed: ${e.message}`); + + return ScryptCancellation.UNCONFIRMED; + } + } + + private async cancelOrder(clOrdId: string, from: string, to: string): Promise { const { symbol } = await this.getTradePair(from, to); + return this.cancelOrderBySymbol(clOrdId, symbol); + } + + private async cancelOrderBySymbol(clOrdId: string, symbol: string): Promise { const newClOrdId = randomUUID(); const cancelData = { @@ -859,11 +1257,25 @@ export class ScryptService extends PricingProvider { ScryptMessageType.ORDER_CANCEL_REQUEST, [cancelData], ScryptMessageType.EXECUTION_REPORT, - (reports) => reports.find((r) => r.OrigClOrdID === clOrdId || r.ClOrdID === newClOrdId) ?? null, + // PendingCancel is the venue saying "working on it", not an answer. Taking the first report that + // merely mentions this order would freeze that intermediate state as the result — and since the + // waiter unsubscribes on its first match, the real terminal report that follows would never be seen. + (reports) => + reports.find( + (r) => + (r.OrigClOrdID === clOrdId || r.ClOrdID === newClOrdId) && r.OrdStatus !== ScryptOrderStatus.PENDING_CANCEL, + ) ?? null, 60000, ); - return report.OrdStatus === ScryptOrderStatus.CANCELED; + // Deliberately not cached under the cancelled order's own id. The venue tags a cancel confirmation with + // the CANCEL request's id, so filing it under the order would make a later status lookup read that + // order as terminally cancelled — and a cleanup cancellation says nothing about the order as a whole: + // its sibling references may still be unsettled and live. A lookup that then reports the order as known + // would take it out of quarantine and let the completion check open a replacement beside them, which is + // the double execution this path exists to prevent. + + return report; } private async editOrder( diff --git a/src/subdomains/core/liquidity-management/adapters/actions/__tests__/scrypt.adapter.spec.ts b/src/subdomains/core/liquidity-management/adapters/actions/__tests__/scrypt.adapter.spec.ts index 03f2765ea9..482ea0f089 100644 --- a/src/subdomains/core/liquidity-management/adapters/actions/__tests__/scrypt.adapter.spec.ts +++ b/src/subdomains/core/liquidity-management/adapters/actions/__tests__/scrypt.adapter.spec.ts @@ -2,6 +2,7 @@ import { createMock } from '@golevelup/ts-jest'; import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.enum'; import { ScryptBalanceTransaction, + ScryptCancellation, ScryptOrderInfo, ScryptOrderStatus, ScryptTransactionStatus, @@ -59,8 +60,8 @@ function createUncertainSellOrder(overrides: Partial = } /** Minimal but fully typed venue order record, so the tests do not have to widen the return type. */ -function venueOrder(id: string, status = ScryptOrderStatus.NEW): ScryptOrderInfo { - return { id, symbol: 'EUR/USDT', side: 'Sell', status, quantity: 1, filledQuantity: 0, remainingQuantity: 1 }; +function venueOrder(id: string, status = ScryptOrderStatus.NEW, symbol = 'EUR/USDT'): ScryptOrderInfo { + return { id, symbol, side: 'Sell', status, quantity: 1, filledQuantity: 0, remainingQuantity: 1 }; } /** Typed action stub — `paramMap` is a getter over `params`, so the raw field is what a fixture sets. */ @@ -68,6 +69,15 @@ function withdrawAction(): LiquidityManagementAction { return Object.assign(new LiquidityManagementAction(), { command: ScryptAdapterCommands.WITHDRAW, params: '{}' }); } +/** Same as {@link withdrawAction}, for the deliberately unregistered command the reconciliation path must + * still handle by name alone (it resolves by system, not by a known command list). */ +function sellIfDeficitAction(paramMap: Record = {}): LiquidityManagementAction { + return Object.assign(new LiquidityManagementAction(), { + command: 'sell-if-deficit', + params: JSON.stringify(paramMap), + }); +} + describe('ScryptAdapter', () => { let adapter: ScryptAdapter; let scryptService: ScryptService; @@ -181,7 +191,8 @@ describe('ScryptAdapter', () => { it('keeps waiting on an aged withdrawal the venue DOES know but has not settled', async () => { // a record without a hash is an observation, not an unknown outcome — quarantining it would only - // bounce the order between reconciliation and the completion check + // bounce the order between reconciliation and the completion check, as long as it is inside the + // 24-hour stuck bound; without this test that bound could shrink far enough to break an ordinary wait jest.spyOn(scryptService, 'getWithdrawalStatus').mockResolvedValue({ id: 'w-inflight', status: ScryptTransactionStatus.COMPLETED, @@ -191,6 +202,18 @@ describe('ScryptAdapter', () => { await expect(adapter.checkCompletion(old)).resolves.toBe(false); }); + it('fails a withdrawal the venue has reported without a transaction hash for over 24 hours', async () => { + // without this bound, a withdrawal stuck at the venue with no txHash would poll forever with no exit + // but a human noticing — exactly the outcome the stuck-withdrawal backstop exists to remove + jest.spyOn(scryptService, 'getWithdrawalStatus').mockResolvedValue({ + id: 'w-stuck', + status: ScryptTransactionStatus.COMPLETED, + }); + const stuck = createWithdrawOrder({ created: new Date(Date.now() - 25 * 60 * 60 * 1000) }); + + await expect(adapter.checkCompletion(stuck)).rejects.toBeInstanceOf(OrderFailedException); + }); + it('still just waits while the withdrawal is young', async () => { jest.spyOn(scryptService, 'getWithdrawalStatus').mockResolvedValue(null); @@ -458,6 +481,240 @@ describe('ScryptAdapter', () => { }); }); + describe('cancelOutstanding', () => { + /** cancelOutstanding derives the trade pair from the rule, so the fixture needs one. */ + function cancellableOrder(overrides: Partial = {}): LiquidityManagementOrder { + return createUncertainSellOrder({ + action: { command: ScryptAdapterCommands.SELL, paramMap: { tradeAsset: 'USDT' } }, + pipeline: { rule: { targetAsset: { dexName: 'EUR' } } }, + ...overrides, + } as Partial); + } + + it('confirms only once the venue has settled every reference the order ever claimed', async () => { + const cancelIfOutstanding = jest + .spyOn(scryptService, 'cancelIfOutstanding') + .mockResolvedValue(ScryptCancellation.SETTLED); + const order = cancellableOrder(); + order.recordSpentCorrelationId('dfx-lm-4711-1'); + + await expect(adapter.cancelOutstanding(order)).resolves.toBe( + 'the venue answered for every reference that nothing is left to execute', + ); + expect(cancelIfOutstanding).toHaveBeenCalledTimes(2); + }); + + it('refuses when a single reference will not settle — that one could still spend the funds', async () => { + jest + .spyOn(scryptService, 'cancelIfOutstanding') + .mockImplementation(async (id: string) => + id === 'dfx-lm-4711' ? ScryptCancellation.UNCONFIRMED : ScryptCancellation.SETTLED, + ); + const order = cancellableOrder(); + order.recordSpentCorrelationId('dfx-lm-4711-1'); + + await expect(adapter.cancelOutstanding(order)).resolves.toBeNull(); + }); + + it('still cancels the older references when the newest one will not settle', async () => { + // leaving the loop at the first refusal would leave exactly those references without an attempt — + // and they are the ones that could be sitting open in the book + const cancelIfOutstanding = jest + .spyOn(scryptService, 'cancelIfOutstanding') + .mockImplementation(async (id: string) => + id === 'dfx-lm-4711-1' ? ScryptCancellation.UNCONFIRMED : ScryptCancellation.SETTLED, + ); + const order = cancellableOrder(); + order.recordSpentCorrelationId('dfx-lm-4711-1'); + + await expect(adapter.cancelOutstanding(order)).resolves.toBeNull(); + expect(cancelIfOutstanding).toHaveBeenCalledWith('dfx-lm-4711-1', 'EUR', 'USDT'); + expect(cancelIfOutstanding).toHaveBeenCalledWith('dfx-lm-4711', 'EUR', 'USDT'); + }); + + it('reconstructs a missing trade correlationId and still asks the venue under dfx-lm-${id}', async () => { + // invariant break stays logged; the reference is deterministic from the order id so a pure lookup is + // safe even when the column was never written + const cancelIfOutstanding = jest + .spyOn(scryptService, 'cancelIfOutstanding') + .mockResolvedValue(ScryptCancellation.SETTLED); + const errorSpy = jest.spyOn(adapter['logger'], 'error').mockImplementation(); + const order = cancellableOrder({ id: 4711, correlationId: null, previousCorrelationIds: null }); + + await expect(adapter.cancelOutstanding(order)).resolves.toBe( + 'the venue answered for every reference that nothing is left to execute', + ); + expect(errorSpy).toHaveBeenCalled(); + expect(cancelIfOutstanding).toHaveBeenCalledWith('dfx-lm-4711', 'EUR', 'USDT'); + }); + + it('treats an executed reference as settled — it cannot execute again either', async () => { + // holding on because something filled would be backwards: the fill is already in the venue's balance, + // and that balance is what the rule replans from, so it plans for what is actually left + jest + .spyOn(scryptService, 'cancelIfOutstanding') + .mockImplementation(async (id: string) => + id === 'dfx-lm-4711-1' ? ScryptCancellation.EXECUTED : ScryptCancellation.SETTLED, + ); + const order = cancellableOrder(); + order.recordSpentCorrelationId('dfx-lm-4711-1'); + + await expect(adapter.cancelOutstanding(order)).resolves.toBe( + 'the venue answered for every reference that nothing is left to execute', + ); + // an abandoned order books no output, so the reference that filled has to be named somewhere + expect(order.errorMessage).toContain('dfx-lm-4711-1'); + }); + + it('does not cancel a withdrawal via cancelIfOutstanding — absence is confirmed instead', async () => { + const cancelIfOutstanding = jest.spyOn(scryptService, 'cancelIfOutstanding'); + jest.spyOn(scryptService, 'confirmWithdrawalAbsent').mockResolvedValue(false); + const order = cancellableOrder({ action: withdrawAction() }); + + await expect(adapter.cancelOutstanding(order)).resolves.toBeNull(); + expect(cancelIfOutstanding).not.toHaveBeenCalled(); + expect(scryptService.confirmWithdrawalAbsent).toHaveBeenCalledWith(order.correlationId); + }); + + it('abandons a withdrawal once the venue answers without naming it', async () => { + const cancelIfOutstanding = jest.spyOn(scryptService, 'cancelIfOutstanding'); + jest.spyOn(scryptService, 'confirmWithdrawalAbsent').mockResolvedValue(true); + const order = cancellableOrder({ action: withdrawAction() }); + + await expect(adapter.cancelOutstanding(order)).resolves.toBe( + 'the venue answered with its transaction history and did not name this withdrawal', + ); + expect(cancelIfOutstanding).not.toHaveBeenCalled(); + expect(scryptService.confirmWithdrawalAbsent).toHaveBeenCalledWith(order.correlationId); + }); + + it('reconstructs a missing withdrawal correlationId and still confirms absence under dfx-lm-${id}', async () => { + const confirmWithdrawalAbsent = jest.spyOn(scryptService, 'confirmWithdrawalAbsent').mockResolvedValue(true); + const errorSpy = jest.spyOn(adapter['logger'], 'error').mockImplementation(); + const order = cancellableOrder({ id: 4711, action: withdrawAction(), correlationId: null }); + + await expect(adapter.cancelOutstanding(order)).resolves.toBe( + 'the venue answered with its transaction history and did not name this withdrawal', + ); + expect(errorSpy).toHaveBeenCalled(); + expect(confirmWithdrawalAbsent).toHaveBeenCalledWith('dfx-lm-4711'); + }); + + it('for an unsupported command asks getOrderStatus and returns a reason when every reference is already terminal', async () => { + // The tradeAsset-derived shortcut is gone (see the comment above the branch in cancelOutstanding): a + // legacy/renamed command always asks the venue first, whether or not paramMap still carries tradeAsset. + const getOrderStatus = jest + .spyOn(scryptService, 'getOrderStatus') + .mockImplementation(async (id: string) => venueOrder(id, ScryptOrderStatus.FILLED)); + const cancelIfOutstandingBySymbol = jest.spyOn(scryptService, 'cancelIfOutstandingBySymbol'); + const order = cancellableOrder({ + action: sellIfDeficitAction({}), + }); + order.recordSpentCorrelationId('dfx-lm-4711-1'); + + await expect(adapter.cancelOutstanding(order)).resolves.toBe( + 'the venue left no reference of this unsupported command able to execute — each is terminal or unknown to it', + ); + expect(cancelIfOutstandingBySymbol).not.toHaveBeenCalled(); + expect(getOrderStatus).toHaveBeenCalledWith('dfx-lm-4711-1'); + expect(getOrderStatus).toHaveBeenCalledWith('dfx-lm-4711'); + }); + + it('for an unsupported command treats venue-unknown (null) as settled and returns a reason', async () => { + // null = venue answered and has no record for that reference — same inference as refusedAsUnknown / + // SCRYPT_UNKNOWN_ORDER on the active path; must not keep the order quarantined. + const getOrderStatus = jest + .spyOn(scryptService, 'getOrderStatus') + .mockImplementation(async (id: string) => + id === 'dfx-lm-4711-1' ? null : venueOrder(id, ScryptOrderStatus.FILLED), + ); + const cancelIfOutstandingBySymbol = jest.spyOn(scryptService, 'cancelIfOutstandingBySymbol'); + const order = cancellableOrder({ + action: sellIfDeficitAction({}), + }); + order.recordSpentCorrelationId('dfx-lm-4711-1'); + + await expect(adapter.cancelOutstanding(order)).resolves.toBe( + 'the venue left no reference of this unsupported command able to execute — each is terminal or unknown to it', + ); + expect(cancelIfOutstandingBySymbol).not.toHaveBeenCalled(); + expect(getOrderStatus).toHaveBeenCalledWith('dfx-lm-4711-1'); + expect(getOrderStatus).toHaveBeenCalledWith('dfx-lm-4711'); + }); + + it('for an unsupported command cancels a non-terminal reference under the symbol the venue supplied, and returns a reason once it settles', async () => { + // FIX B: the venue's own getOrderStatus reply names the symbol, so a non-terminal legacy reference is + // no longer just polled — it is actively cancelled under exactly that symbol, no tradeAsset needed. + jest + .spyOn(scryptService, 'getOrderStatus') + .mockImplementation(async (id: string) => + id === 'dfx-lm-4711-1' + ? venueOrder(id, ScryptOrderStatus.NEW, 'XRP/USDT') + : venueOrder(id, ScryptOrderStatus.FILLED), + ); + const cancelIfOutstandingBySymbol = jest + .spyOn(scryptService, 'cancelIfOutstandingBySymbol') + .mockResolvedValue(ScryptCancellation.SETTLED); + const order = cancellableOrder({ + action: sellIfDeficitAction({}), + }); + order.recordSpentCorrelationId('dfx-lm-4711-1'); + + await expect(adapter.cancelOutstanding(order)).resolves.toBe( + 'the venue left no reference of this unsupported command able to execute — each is terminal or unknown to it', + ); + expect(cancelIfOutstandingBySymbol).toHaveBeenCalledWith('dfx-lm-4711-1', 'XRP/USDT'); + }); + + it('for an unsupported command records a reference the symbol-based cancel reports as executed', async () => { + // Mirrors the known-command path: a fill is not a reason to hold on, but it is worth naming so it can + // be reconciled against the venue balance. + jest + .spyOn(scryptService, 'getOrderStatus') + .mockImplementation(async (id: string) => venueOrder(id, ScryptOrderStatus.PARTIALLY_FILLED, 'XRP/USDT')); + jest.spyOn(scryptService, 'cancelIfOutstandingBySymbol').mockResolvedValue(ScryptCancellation.EXECUTED); + const order = cancellableOrder({ + action: sellIfDeficitAction({}), + }); + + await adapter.cancelOutstanding(order); + + expect(order.errorMessage).toContain('dfx-lm-4711'); + }); + + it('for an unsupported command keeps the order quarantined when the symbol-based cancel stays UNCONFIRMED', async () => { + jest + .spyOn(scryptService, 'getOrderStatus') + .mockImplementation(async (id: string) => venueOrder(id, ScryptOrderStatus.NEW, 'XRP/USDT')); + const cancelIfOutstandingBySymbol = jest + .spyOn(scryptService, 'cancelIfOutstandingBySymbol') + .mockResolvedValue(ScryptCancellation.UNCONFIRMED); + const order = cancellableOrder({ + action: sellIfDeficitAction({}), + }); + order.recordSpentCorrelationId('dfx-lm-4711-1'); + + await expect(adapter.cancelOutstanding(order)).resolves.toBeNull(); + expect(cancelIfOutstandingBySymbol).toHaveBeenCalled(); + }); + + it('for an unsupported command returns null when any reference is unreachable, without attempting a cancel', async () => { + // undefined (fetch failure / catch) is not a venue answer — keep waiting; must stay distinct from null. + jest.spyOn(scryptService, 'getOrderStatus').mockImplementation(async (id: string) => { + if (id === 'dfx-lm-4711-1') throw new Error('Connection closed'); + return venueOrder(id, ScryptOrderStatus.FILLED); + }); + const cancelIfOutstandingBySymbol = jest.spyOn(scryptService, 'cancelIfOutstandingBySymbol'); + const order = cancellableOrder({ + action: sellIfDeficitAction({}), + }); + order.recordSpentCorrelationId('dfx-lm-4711-1'); + + await expect(adapter.cancelOutstanding(order)).resolves.toBeNull(); + expect(cancelIfOutstandingBySymbol).not.toHaveBeenCalled(); + }); + }); + describe('resolveUncertainOrder', () => { it('reports SENT when the venue knows the reference', async () => { jest.spyOn(scryptService, 'getOrderStatus').mockResolvedValue(venueOrder('dfx-lm-4711')); @@ -476,6 +733,21 @@ describe('ScryptAdapter', () => { await expect(adapter.resolveUncertainOrder(ancient)).resolves.toBe(UncertainOrderResolution.UNRESOLVED); }); + it.each([[undefined], [null]])( + 'reconstructs a missing reference %p as dfx-lm-${id} and still asks the venue', + async (correlationId: string | null | undefined) => { + // invariant break stays logged; the reconstructed id is deterministic from the order row so a pure + // lookup is safe, and leaving the order for an operator forever is not allowed + const getOrderStatus = jest.spyOn(scryptService, 'getOrderStatus').mockResolvedValue(null); + const errorSpy = jest.spyOn(adapter['logger'], 'error').mockImplementation(); + const order = createUncertainSellOrder({ id: 4711, correlationId, previousCorrelationIds: null }); + + await expect(adapter.resolveUncertainOrder(order)).resolves.toBe(UncertainOrderResolution.UNRESOLVED); + expect(errorSpy).toHaveBeenCalled(); + expect(getOrderStatus).toHaveBeenCalledWith('dfx-lm-4711', expect.any(Date)); + }, + ); + it('reports UNAVAILABLE when the lookup itself fails — no question reached the venue', async () => { jest.spyOn(scryptService, 'getOrderStatus').mockRejectedValue(new Error('Connection closed')); @@ -484,12 +756,6 @@ describe('ScryptAdapter', () => { ); }); - it('stays UNRESOLVED when no reference was ever reserved', async () => { - const order = createUncertainSellOrder({ correlationId: undefined }); - - await expect(adapter.resolveUncertainOrder(order)).resolves.toBe(UncertainOrderResolution.UNRESOLVED); - }); - it('uses the withdrawal lookup for withdraw orders', async () => { jest.spyOn(scryptService, 'findWithdrawal').mockResolvedValue(null); const order = createUncertainSellOrder({ @@ -550,7 +816,8 @@ describe('ScryptAdapter', () => { it('stays quarantined when a claimed replacement is not (yet) visible, even if the predecessor is', async () => { // an accepted replacement may lag in the venue's view; falling back to the order it replaced would - // report SENT on a superseded reference and leave the live replacement untracked + // report SENT on a superseded reference and leave the live replacement untracked. + // jest .spyOn(scryptService, 'getOrderStatus') .mockImplementation(async (id: string) => (id === 'dfx-lm-4711' ? venueOrder(id) : null)); @@ -560,6 +827,16 @@ describe('ScryptAdapter', () => { await expect(adapter.resolveUncertainOrder(order)).resolves.toBe(UncertainOrderResolution.UNRESOLVED); }); + it('reports a single unknown reference as UNRESOLVED — nothing was left unasked', async () => { + // the ordinary case: one attempt, the venue answered about it, and there is no older reference the + // lookup skipped. That is a complete answer, and the caller's bound may act on it. + jest.spyOn(scryptService, 'getOrderStatus').mockResolvedValue(null); + + await expect(adapter.resolveUncertainOrder(createUncertainSellOrder())).resolves.toBe( + UncertainOrderResolution.UNRESOLVED, + ); + }); + it('falls back to the predecessor only after the replacement was explicitly rejected', async () => { jest .spyOn(scryptService, 'getOrderStatus') @@ -608,5 +885,14 @@ describe('ScryptAdapter', () => { expect(since?.getTime()).toBeGreaterThanOrEqual(earliest.getTime()); expect(since?.getTime()).toBeLessThanOrEqual(latest.getTime()); }); + + it('resolves an unknown command on the command-independent trade path', async () => { + jest.spyOn(scryptService, 'getOrderStatus').mockResolvedValue(venueOrder('dfx-lm-4711')); + const order = createUncertainSellOrder({ + action: sellIfDeficitAction(), + }); + + await expect(adapter.resolveUncertainOrder(order)).resolves.toBe(UncertainOrderResolution.SENT); + }); }); }); diff --git a/src/subdomains/core/liquidity-management/adapters/actions/scrypt.adapter.ts b/src/subdomains/core/liquidity-management/adapters/actions/scrypt.adapter.ts index 2d4f2c155d..27723d8e04 100644 --- a/src/subdomains/core/liquidity-management/adapters/actions/scrypt.adapter.ts +++ b/src/subdomains/core/liquidity-management/adapters/actions/scrypt.adapter.ts @@ -1,6 +1,8 @@ import { Injectable } from '@nestjs/common'; import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.enum'; import { + isTerminalScryptOrderStatus, + ScryptCancellation, ScryptOrderInfo, ScryptOrderSide, ScryptOrderStatus, @@ -11,6 +13,7 @@ import { isVenueRejection, ScryptAmendRejectedError, ScryptOrderNotFoundError, + ScryptOrderStuckPendingError, ScryptUnconfirmedWriteError, } from 'src/integration/exchange/services/scrypt-websocket-connection'; import { ScryptService } from 'src/integration/exchange/services/scrypt.service'; @@ -41,11 +44,41 @@ const SCRYPT_CORRELATION_PREFIX = 'dfx-lm-'; /** * How long an acknowledged order may stay unobservable before it is quarantined rather than polled again. * - * Matches the age at which the venue lookup itself gives up on finding an order, so both routes out of a - * silent order agree. Quarantine is not a verdict — the order is still not declared failed — it only moves it - * somewhere a human can act on. + * Five minutes, matching the age at which the venue lookup itself gives up on finding an order + * (`ORDER_LOST_AFTER_MINUTES`), so both routes out of a silent order still agree. Together with the + * five-minute abandon bound for these commands that is the ten-minute ceiling the orderer set — nominal, + * and conditional on the venue answering at all: the pass runs on a ten-second cron with jitter, and a + * venue that cannot be reached holds the order past that point, because the exit rests on its answer + * rather than on the clock. See ABANDON_UNCERTAIN_MINUTES.VENUE_WITHDRAWAL for the same bound argued + * from the other end. A venue record is written when a request is ACCEPTED, not when it finishes — so its + * absence after five minutes says the acceptance is in doubt, which is independent of a withdrawal itself + * being allowed to take hours. Quarantine is not a verdict — the order is still not declared failed here — it only + * moves it where the caller's bound can attempt an automatic exit (cancel every trade reference, or confirm + * a withdrawal is unnamed in the venue's history reply). An operator can still release sooner as a shortcut; the human + * is not the rule path for either command. */ -const SCRYPT_UNOBSERVABLE_QUARANTINE_MINUTES = 60; +const SCRYPT_UNOBSERVABLE_QUARANTINE_MINUTES = 5; + +/** + * Backstop against a withdrawal whose venue record never gains a transaction hash — not a bound against a + * merely slow one. + * + * `ScryptTransactionStatus` (scrypt.dto.ts) declares only COMPLETED | FAILED | REJECTED, and FAILED/REJECTED + * are already handled above this branch, so what actually reaches here is either a Completed record without + * a hash or a status outside that declared enum — not a documented "in progress" state either way. What is + * certain is only the absence: a record without a `txHash` counts as not finished, whatever status the venue + * attaches to it. Without a ceiling here the only exit left would be a human noticing, and that is exactly + * the outcome this system exists to remove. Measured over 60 days in production, withdrawals took a median + * of 6.6 minutes to complete, a p95 of 90 minutes, and the single slowest observed run took 336 minutes (5.6 + * hours). 24 hours is a good four times that worst case — far outside anything slowness alone could explain. + * + * The exit is failing the order so the rule can replan; if the venue pays out afterwards regardless, that + * is only an internal rebooking, because every Scrypt withdrawal address is DFX's own. Deliberately NOT + * folded into the quarantine bound above: an order with a venue record already on file makes the absence + * proof `confirmWithdrawalAbsent` relies on unreachable, and the order would only bounce between + * reconciliation and this completion check instead of ever resolving. + */ +const SCRYPT_WITHDRAWAL_STUCK_AFTER_MINUTES = 24 * 60; @Injectable() export class ScryptAdapter extends LiquidityActionAdapter { @@ -230,12 +263,22 @@ export class ScryptAdapter extends LiquidityActionAdapter { // No record at all, past the age at which the venue is considered to have lost it: we cannot tell // whether this withdrawal happened, and the manual path only accepts quarantined orders, so leaving it // here would mean no way out at all. A record WITHOUT a hash is different — that is an observation, the - // withdrawal is simply still in flight, and quarantining it would only bounce it back and forth. + // withdrawal is simply still in flight, but only up to SCRYPT_WITHDRAWAL_STUCK_AFTER_MINUTES: short of + // that bound quarantining it would only bounce it back and forth between reconciliation and this check. if (!withdrawal && Util.minutesDiff(order.created) > SCRYPT_UNOBSERVABLE_QUARANTINE_MINUTES) throw new OrderOutcomeUnknownException( `Scrypt has no record of withdrawal ${correlationId} after more than ${SCRYPT_UNOBSERVABLE_QUARANTINE_MINUTES} minutes`, ); + if (withdrawal) { + const ageMinutes = Util.minutesDiff(order.created); + if (ageMinutes > SCRYPT_WITHDRAWAL_STUCK_AFTER_MINUTES) + throw new OrderFailedException( + `Withdrawal ${correlationId} is ${Math.round(ageMinutes)} minutes old: the venue reports status ` + + `${withdrawal.status} without a transaction hash, longer than the ${SCRYPT_WITHDRAWAL_STUCK_AFTER_MINUTES}-minute stuck bound`, + ); + } + this.logger.verbose(`No withdrawal id for id ${correlationId} at ${this.scryptService.name} found`); return false; } @@ -260,6 +303,192 @@ export class ScryptAdapter extends LiquidityActionAdapter { return this.checkTradeCompletion(order, tradeAsset, asset); } + /** + * Make sure nothing this order could still have live can execute, so the caller may give it up. + * + * For trades: every reference the row ever claimed — sent or merely reserved — not just the current one. + * The reason an order gets here is that at least one of them has an outcome nobody could observe, and an + * unobserved reference is precisely the one that might be sitting in the book. These are GTC orders — + * nothing expires them — so age is no argument at all, and the only way to know a reference cannot fill is + * to have the venue say so. All-or-nothing on purpose: one reference the venue would not settle is enough + * to keep the whole order quarantined, because the funds a rule would get back are the same funds that + * reference could still spend. Every reference is still asked about — a refusal on one is no reason to + * leave the others without an attempt. An unsupported/legacy command cancels the same way; it just names + * its cancel symbol from the venue's own order-status reply instead of deriving one locally, so there is + * no command for which "no symbol" blocks this exit. + * + * For withdrawals: Scrypt has no cancel operation. The exit rests on absence — the venue answered + * successfully and that reply has no record of this reference. Deliberately no completeness check on that + * reply: withdrawal destinations are DFX-owned, so a truncated answer costs an internal rebooking, while a + * check demanding local anchors would strand the order forever (see `confirmWithdrawalAbsent`). So this is + * weaker than "the request never arrived" AND weaker than "the history was complete" — it is only "the + * venue answered and did not name it". Exactly three answers return null and leave the order quarantined + * for another pass: a lookup that failed, a reply that does name the reference, or the reference surfacing + * in the live cache while the lookup was open. An empty reply is none of them — a reference cannot be in a + * history with no rows, so that confirms absence. + * + * Returns the reason string the caller records on abandon, or null when nothing is settled yet. + * + * The replan that follows reads the venue's balance, which is pushed rather than polled, so a fill that + * has only just landed may not be in it yet. In practice that push follows a fill within seconds and the + * abandoned order is the slower half of the race, but it is a window rather than a guarantee — worth + * knowing if a rule is ever seen planning against a balance that looks one fill stale. + */ + async cancelOutstanding(order: LiquidityManagementOrder): Promise { + // Reconstruct a missing reference from the order id alone (see reserveCorrelationId). The error log + // remains: reserve-before-send should make this state impossible. Reconstruction is still safe to ask + // about — the id is deterministic from the row, has no random component, and a pure venue lookup under + // it is harmless if nothing was ever sent under that name. + if (!order.correlationId) { + this.logger.error( + `Order ${order.id}: Scrypt order reached cancelOutstanding with no correlationId — reserve-before-send should make this impossible`, + ); + order.reserveCorrelationId(this.reserveCorrelationId(order)); + } + + if (order.action.command === ScryptAdapterCommands.WITHDRAW) { + const absent = await this.scryptService.confirmWithdrawalAbsent(order.correlationId); + return absent ? 'the venue answered with its transaction history and did not name this withdrawal' : null; + } + + const references = this.attemptedReferencesNewestFirst(order); + + // Nothing ever went out under a reference, so the venue cannot confirm anything about this order — and + // an empty loop would otherwise fall through to "all settled" without a single question asked. Absent + // evidence this must hold the order, never release it. Since reserve-before-send a Scrypt order should + // not reach quarantine without a reference; this is an invariant break, not a planned wait. + if (!references.length) { + this.logger.error( + `Order ${order.id}: Scrypt trade reached cancelOutstanding with no references — reserve-before-send should make this impossible`, + ); + return null; + } + + // A command no longer in ScryptAdapterCommands (rename/removal) still reaches this adapter via + // getReconciliationIntegration. There used to be a second path here for such a command when its + // paramMap still carried a `tradeAsset` — but that was only ever a shortcut to a symbol, and a shortcut + // that could be wrong: a `tradeAsset` in a stale paramMap is not a guarantee the command was ever a + // plain sell/buy, "SELL vs. everything else" is not a real side inference for a command that, by + // definition, is not SELL, and getTradePair reads live security configuration that a delisted or + // renamed pair could no longer match even though the order itself is still open at the venue under its + // original symbol. None of that guessing is needed: the venue's own order-status reply already names + // the symbol a reference lives under (ScryptOrderInfo.symbol), so every reference the venue can still + // show us is cancellable through the symbol it hands back, with no local trade-pair derivation at all. + // One way for every unsupported command, not two: ask first, storno under the venue's own symbol only + // if the answer is non-terminal. `null` vs `undefined` is the whole point here (same distinction as + // adoptLiveReplacement): null = venue answered and has no record; undefined = could not be asked. + const knownCommands = Object.values(ScryptAdapterCommands) as string[]; + const isKnownCommand = knownCommands.includes(order.action.command); + + if (!isKnownCommand) { + const executed: CorrelationId[] = []; + let unsettled = 0; + + for (const reference of references) { + const info = await this.scryptService.getOrderStatus(reference).catch(() => undefined); + + if (info === undefined) { + this.logger.warn( + `Order ${order.id}: unsupported command ${order.action.command} — reference ${reference} is unreachable; keeping the order quarantined`, + ); + return null; + } + + if (info === null) { + // Venue answered: no record for this reference. Same inference as cancelIfOutstanding / + // refusedAsUnknown (SCRYPT_UNKNOWN_ORDER): nothing left that can execute under it. + this.logger.info( + `Order ${order.id}: unsupported command ${order.action.command} — reference ${reference} is unknown to the venue — treated as settled`, + ); + continue; + } + + if (isTerminalScryptOrderStatus(info.status)) continue; + + // Non-terminal, and the venue just named the symbol it lives under in the very same reply — storno + // it under exactly that symbol. Evaluated the same way the active path below evaluates a cancel: + // SETTLED moves on to the next reference, EXECUTED is recorded so the fill can be reconciled, + // anything else keeps the whole order quarantined (a single unsettled reference could still spend + // the funds). + const outcome = await this.scryptService.cancelIfOutstandingBySymbol(reference, info.symbol); + + if (outcome === ScryptCancellation.SETTLED) continue; + + if (outcome === ScryptCancellation.EXECUTED) { + executed.push(reference); + continue; + } + + unsettled++; + this.logger.warn( + `Order ${order.id}: unsupported command ${order.action.command} — Scrypt would not settle ${reference}, so it may still execute — keeping the order quarantined`, + ); + } + + if (unsettled) return null; + + if (executed.length) + order.errorMessage = `${order.errorMessage} (executed at Scrypt under ${executed.join(', ')})`; + + this.logger.info( + `Order ${order.id}: venue leaves no reference of unsupported command ${order.action.command} able to execute — each is terminal or unknown to it${ + executed.length ? `; ${executed.join(', ')} had filled` : '' + }`, + ); + return 'the venue left no reference of this unsupported command able to execute — each is terminal or unknown to it'; + } + + // Known command: from/to is derived directly from the rule, so the venue's cancel-and-report is asked + // for every reference straight away — no separate status lookup needed first (contrast the branch + // above, which cannot derive a trade pair locally and asks first for that reason). + const { tradeAsset } = this.parseTradeParams(order.action.paramMap); + const asset = order.pipeline.rule.targetAsset.dexName; + const [from, to] = order.action.command === ScryptAdapterCommands.SELL ? [asset, tradeAsset] : [tradeAsset, asset]; + + const executed: CorrelationId[] = []; + let unsettled = 0; + + for (const reference of references) { + const outcome = await this.scryptService.cancelIfOutstanding(reference, from, to); + + // Cancelled and executed both answer the only question that matters here: can this reference still + // execute? It cannot — one because it was called off, the other because it ran to a terminal state. + // A fill is not a reason to hold on: it is already in the venue's balance, and that balance is what + // the rule replans from, so it plans for what is actually left rather than for what this row + // believed. Which reference filled is recorded on the order below, since the row itself no longer + // carries that after being abandoned. + if (outcome === ScryptCancellation.SETTLED) continue; + + if (outcome === ScryptCancellation.EXECUTED) { + executed.push(reference); + continue; + } + + // Counted, not returned on. Leaving the loop here would leave every older reference without so much + // as a cancellation attempt — and those are exactly the ones that can sit open in the book while the + // newest keeps refusing to settle. Ask about all of them, then decide. + unsettled++; + this.logger.warn( + `Order ${order.id}: Scrypt would not settle ${reference}, so it may still execute — keeping the order quarantined`, + ); + } + + if (unsettled) return null; + + // Which reference filled is the one thing an abandoned order can no longer say for itself — its status + // becomes FAILED and it books no output. The venue's own transaction record carries the money side, but + // tying that back to this row afterwards needs the reference named somewhere, so name it here. + if (executed.length) order.errorMessage = `${order.errorMessage} (executed at Scrypt under ${executed.join(', ')})`; + + this.logger.info( + `Order ${order.id}: Scrypt answered for every reference that nothing is left to execute${ + executed.length ? `; ${executed.join(', ')} had filled` : '' + }`, + ); + + return 'the venue answered for every reference that nothing is left to execute'; + } + private async checkTradeCompletion(order: LiquidityManagementOrder, from: string, to: string): Promise { // Before anything may write again: a previous pass may have had its replacement accepted and then failed // to record it, leaving this row pointing at the predecessor the venue has already cancelled. Restarting @@ -318,9 +547,19 @@ export class ScryptAdapter extends LiquidityActionAdapter { } // The venue once acknowledged this order and now cannot find it. That is not a failure — it may have - // filled or been cancelled outside our view — so it goes to a human instead of releasing the rule. + // filled or been cancelled outside our view — so it goes to quarantine instead of releasing the rule. + // From there the caller's bound attempts a cancellation whose confirmation ends it; an operator can + // still release sooner as a shortcut, but is not the rule path. if (e instanceof ScryptOrderNotFoundError) throw new OrderOutcomeUnknownException(e.message); + // Not a blind spot: asked to cancel a reference past its bound, the venue settled it. See + // {@link ScryptOrderStuckPendingError} for what that settlement rests on: a terminal cancel with + // nothing filled, or the venue no longer recognising the reference — the latter an inference from + // SCRYPT_UNKNOWN_ORDER rather than a directly observed fact. Either reading lands at the same + // conclusion, so the order fails outright and the rule may replan straight away instead of waiting on + // a bound already spent. + if (e instanceof ScryptOrderStuckPendingError) throw new OrderFailedException(e.message); + // A rejection is a reply: the venue reached a verdict, so the order really did end. if (isVenueRejection(e)) throw new OrderFailedException(e.message); @@ -355,6 +594,11 @@ export class ScryptAdapter extends LiquidityActionAdapter { * is a barrier rather than something to step past — the reference is recorded BEFORE the request leaves, * so one the venue does not show may still be live there, and carrying on with the predecessor would put a * second request next to it. + * + * The barrier is meant to hold. What eventually ends such an order is not this method giving way, but the + * caller cancelling every reference it ever claimed — once the venue answers that none of them can + * execute, the + * claim is settled and there is nothing left to block on. */ private async adoptLiveReplacement(order: LiquidityManagementOrder): Promise { const currentAttempt = this.attemptNumber(order, order.correlationId); @@ -398,8 +642,9 @@ export class ScryptAdapter extends LiquidityActionAdapter { * Waiting is the safe answer — writing against an order whose true state is unknown is how a second * request against the same funds happens. But not forever: the manual path only accepts quarantined * orders, so an order nobody can ever observe would poll for good with no way out at all. Past the same - * age at which the venue itself is considered to have lost an order, it goes to a human instead — still - * not declared failed. + * age at which the venue itself is considered to have lost an order, it goes to quarantine instead — + * still not declared failed here. From there the caller's bound ends it via cancellation confirmation; + * an operator can still release sooner as a shortcut. */ private waitOrQuarantine(order: LiquidityManagementOrder, reason: string): boolean { if (Util.minutesDiff(order.created) > SCRYPT_UNOBSERVABLE_QUARANTINE_MINUTES) @@ -413,12 +658,13 @@ export class ScryptAdapter extends LiquidityActionAdapter { } /** - * Every reference this order has actually put on the wire, newest first. + * Every reference this order has claimed — sent or merely reserved — newest first. * * Ordered by the attempt suffix rather than by storage order, so it does not depend on how the list was * assembled. Deliberately does NOT include the next reference: that one has not been sent, and looking for - * it would stop the search on an absence that means nothing — leaving the reference that WAS sent unchecked - * and the order quarantined for good. + * it would stop the search on an absence that means nothing — leaving the reference that WAS sent + * unchecked, and the order quarantined until a cancellation the caller's bound attempts settles a request + * that may well be live. */ private attemptedReferencesNewestFirst(order: LiquidityManagementOrder): CorrelationId[] { return [...order.allCorrelationIds].sort((a, b) => this.attemptNumber(order, b) - this.attemptNumber(order, a)); @@ -582,21 +828,37 @@ export class ScryptAdapter extends LiquidityActionAdapter { // follows rejects the pending request with a generic message that says nothing about whether the venue // acted on them. Both become unknown outcomes. // - // Over-classifying costs an operator a look at the venue; under-classifying is what moved money without - // a record. Since absence at the venue is not proof, such an order waits for a human rather than - // resolving itself — deliberately the expensive direction, because the cheap one is the dangerous one. + // Over-classifying costs another automatic pass against the venue; under-classifying is what moved money + // without a record. Since absence at the venue is not proof, such an order waits on the venue — for a + // cancellation confirmation (trade) or a history reply that does not name it (withdraw) once its bound + // is reached — rather than resolving itself here, deliberately the expensive direction, because the cheap + // one is the dangerous one. An operator can still release sooner as a shortcut. return new OrderOutcomeUnknownException(`Scrypt gave no confirmed outcome for the ${description}: ${e.message}`); } /** * Ask Scrypt what happened to a quarantined order. Observes only — never re-sends. * - * Can only ever confirm a positive: Scrypt has no terminal "this reference was never accepted" reply, so - * a missing record leaves the order quarantined for a human rather than releasing its rule. + * Only a matched reference can confirm a positive. A missing record confirms nothing on its own — Scrypt + * has no terminal "this reference was never accepted" reply — so it leaves the order quarantined. What + * ends it is not this method concluding anything, but the caller settling every reference the order ever + * claimed: cancel confirmation for trades, or an unnaming history reply for withdrawals. Once + * the venue answers that nothing can still execute, giving up is a fact rather than a guess. An explicit + * rejection of every attempted trade reference is the one negative that settles here, and returns NOT_SENT. */ async resolveUncertainOrder(order: LiquidityManagementOrder): Promise { + // Reconstruct a missing reference from the order id alone (see reserveCorrelationId). The error log + // remains: reserve-before-send should make this state impossible. Reconstruction is still safe — the id + // is deterministic from the row, has no random component, and a pure venue lookup under it is harmless + // if nothing was ever sent under that name. Without it the order would wait on an operator forever. + if (!order.correlationId) { + this.logger.error( + `Order ${order.id}: Scrypt order reached resolveUncertainOrder with no correlationId — reserve-before-send should make quarantining without a reference impossible; this is a bug`, + ); + order.reserveCorrelationId(this.reserveCorrelationId(order)); + } + const { correlationId } = order; - if (!correlationId) return UncertainOrderResolution.UNRESOLVED; let allAttemptsRejected = false; @@ -608,6 +870,11 @@ export class ScryptAdapter extends LiquidityActionAdapter { return UncertainOrderResolution.SENT; } } else { + // Trade path (and any command that is not WITHDRAW, including unknown/renamed commands): works for + // every command because it never consults order.action.command or parseTradeParams — only + // attemptedReferencesNewestFirst and getOrderStatus. The outer branch is `command === WITHDRAW`, + // which is false for an unknown command, so those fall here correctly and stay command-independent. + // // Newest first. A replacement supersedes the order it replaced, and the replaced one usually still // exists at the venue in a cancelled state — checking oldest first would match that, report SENT and // leave the live replacement untracked while the completion check polls a superseded reference. @@ -629,11 +896,12 @@ export class ScryptAdapter extends LiquidityActionAdapter { this.logger.warn( `Scrypt does not (yet) know reference ${candidate} for order ${order.id} — keeping it quarantined`, ); + return UncertainOrderResolution.UNRESOLVED; } - // A refused replacement never took effect and leaves its predecessor live. This is the only case in - // which an older reference may be considered. + // A refused replacement never took effect and leaves its predecessor live. This is the only case + // with an explicit reply that reaches an older reference — the timeout above is the other way. if (info.status === ScryptOrderStatus.REJECTED) { order.recordSpentCorrelationId(candidate); rejectedCount++; @@ -660,8 +928,14 @@ export class ScryptAdapter extends LiquidityActionAdapter { // Absence is NOT proof. A snapshot without the reference may simply predate the venue registering it, // and Scrypt offers no terminal "this was never accepted" acknowledgement to rely on. Concluding - // otherwise is what would let the rule reissue a request that later materialises — so the order stays - // quarantined for a human, and the rule stays blocked, which is the safe direction. + // otherwise is what would let the rule reissue a request that later materialises — so this reports + // only what it saw, and never resolves the order on absence alone. + // + // The caller bounds the wait: an order stuck here long enough gets an automatic exit attempt (cancel + // every trade reference, or a history reply that does not name the withdrawal) rather than being held for + // an operator who may never come, and only that attempt's confirmation abandons it. Both belong there, + // not here — this method's job is to report what the venue said, not to decide how long a rule may stay + // blocked or when giving up is safe. this.logger.warn( `Scrypt still has no record of reference ${correlationId} for order ${order.id} — keeping it quarantined`, ); diff --git a/src/subdomains/core/liquidity-management/entities/__tests__/liquidity-management-order.entity.spec.ts b/src/subdomains/core/liquidity-management/entities/__tests__/liquidity-management-order.entity.spec.ts index 3e7995c837..c4fbcefe48 100644 --- a/src/subdomains/core/liquidity-management/entities/__tests__/liquidity-management-order.entity.spec.ts +++ b/src/subdomains/core/liquidity-management/entities/__tests__/liquidity-management-order.entity.spec.ts @@ -26,6 +26,180 @@ describe('LiquidityManagementOrder', () => { }); }); + describe('unresolvableTooLong', () => { + function quarantined(minutes?: number, command = 'sell', system = 'Scrypt'): LiquidityManagementOrder { + return Object.assign(new LiquidityManagementOrder(), { + status: LiquidityManagementOrderStatus.UNCERTAIN, + updated: minutes == null ? undefined : minutesAgo(minutes), + action: { system, command }, + }); + } + + it('is false without a quarantine timestamp — a missing date reads as the epoch, which would fire at once', () => { + expect(quarantined(undefined).unresolvableTooLong()).toBe(false); + }); + + it.each(['sell', 'buy'])('applies the short trade bound to scrypt/%s', (command) => { + expect(quarantined(4, command).unresolvableTooLong()).toBe(false); + expect(quarantined(6, command).unresolvableTooLong()).toBe(true); + }); + + it('applies the short bound to scrypt/withdraw — the venue answers about its own history', () => { + // Not derived from how long withdrawals take (they reach hours): the bound only applies while the + // venue does not name the reference, and it exists to hold the ten-minute ceiling together with the + // unobservable window. + expect(quarantined(4, 'withdraw').unresolvableTooLong()).toBe(false); + expect(quarantined(6, 'withdraw').unresolvableTooLong()).toBe(true); + }); + + it('leaves the long bound on a transfer at a venue that cannot be asked', () => { + // Lowering the shared transfer bound would have reached bridges and mints too, whose adapters cannot + // attempt any exit — they would only be polled more often for nothing. + expect(quarantined(11 * 60, 'withdraw', 'Kraken').unresolvableTooLong()).toBe(false); + expect(quarantined(13 * 60, 'withdraw', 'Kraken').unresolvableTooLong()).toBe(true); + }); + + it('applies the long bound to a command that only looks like a trade elsewhere', () => { + // an on-chain swap is not a book match, whatever it is called + expect(quarantined(30, 'sell', 'DfxDex').unresolvableTooLong()).toBe(false); + }); + + it('applies the long bound to anything unrecognised', () => { + expect(quarantined(30, 'some-new-command', 'SomeNewSystem').unresolvableTooLong()).toBe(false); + }); + + it('falls back to created when updated is missing — bound can still expire', () => { + // deliberate fallback: created is older-or-equal, so the bound fires earlier, not later; abandon still + // needs venue confirmation, so an earlier cleanup attempt is safe + const order = Object.assign(new LiquidityManagementOrder(), { + status: LiquidityManagementOrderStatus.UNCERTAIN, + updated: undefined, + created: minutesAgo(6), + action: { system: 'Scrypt', command: 'sell' }, + }); + expect(order.unresolvableTooLong()).toBe(true); + }); + + it('stays false when both updated and created are missing', () => { + const order = Object.assign(new LiquidityManagementOrder(), { + status: LiquidityManagementOrderStatus.UNCERTAIN, + updated: undefined, + created: undefined, + action: { system: 'Scrypt', command: 'sell' }, + }); + expect(order.unresolvableTooLong()).toBe(false); + }); + }); + + describe('getAbandonableAt', () => { + function quarantined(minutes?: number, command = 'sell', system = 'Scrypt'): LiquidityManagementOrder { + return Object.assign(new LiquidityManagementOrder(), { + status: LiquidityManagementOrderStatus.UNCERTAIN, + updated: minutes == null ? undefined : minutesAgo(minutes), + action: { system, command }, + }); + } + + /** Milliseconds from now until the order may be given up; negative once it already may be. */ + function headroom(order: LiquidityManagementOrder): number { + const at = order.getAbandonableAt(); + if (!at) throw new Error('expected a deadline'); + + return at.getTime() - Date.now(); + } + + it('is null without a timestamp — no deadline to respect constrains nobody', () => { + expect(quarantined(undefined).getAbandonableAt()).toBeNull(); + }); + + it('falls back to created when updated is missing and returns a Date', () => { + const order = Object.assign(new LiquidityManagementOrder(), { + status: LiquidityManagementOrderStatus.UNCERTAIN, + updated: undefined, + created: minutesAgo(6), + action: { system: 'Scrypt', command: 'sell' }, + }); + expect(order.getAbandonableAt()).toBeInstanceOf(Date); + }); + + it('is null when both updated and created are missing', () => { + const order = Object.assign(new LiquidityManagementOrder(), { + status: LiquidityManagementOrderStatus.UNCERTAIN, + updated: undefined, + created: undefined, + action: { system: 'Scrypt', command: 'sell' }, + }); + expect(order.getAbandonableAt()).toBeNull(); + }); + + it('falls the trade bound after the moment quarantine began', () => { + // 2 of the 5 minutes spent, so 3 left; tolerance covers the clock moving during the test + expect(headroom(quarantined(2))).toBeGreaterThan(2.9 * 60_000); + expect(headroom(quarantined(2))).toBeLessThanOrEqual(3 * 60_000); + }); + + it('lies in the past once the bound has passed, and says by how much', () => { + // The sign is the point: a duration clamped at zero cannot say whether a wait started before the + // deadline, which is exactly what a throttle has to know after it. + expect(headroom(quarantined(6))).toBeLessThan(0); + expect(headroom(quarantined(6))).toBeGreaterThan(-1.1 * 60_000); + expect(headroom(quarantined(60 * 24))).toBeLessThan(-23 * 60 * 60_000); + }); + + it('falls the long bound for a transfer nobody can ask about', () => { + // an hour into a twelve-hour bound leaves eleven + expect(headroom(quarantined(60, 'withdraw', 'Kraken'))).toBeGreaterThan(10.9 * 60 * 60_000); + expect(headroom(quarantined(60, 'withdraw', 'Kraken'))).toBeLessThanOrEqual(11 * 60 * 60_000); + }); + + it('turns over at the bound itself, not a tick later', () => { + // The one instant a live clock cannot be asked about: elapsed exactly equal to the bound. Anything built + // from a real `Date.now()` is already a fraction past it, where `>` and `>=` agree — so the clock is held + // still. With `>` the pass would run (the deadline has arrived) and abandon nothing (not yet past it), + // re-stamp its cooldown, and the order would wait out another interval: five minutes became six. + jest.useFakeTimers(); + try { + const order = quarantined(5); + expect(headroom(order)).toBe(0); + expect(order.unresolvableTooLong()).toBe(true); + + // and one millisecond short of it, both still say no + const justInside = Object.assign(new LiquidityManagementOrder(), { + status: LiquidityManagementOrderStatus.UNCERTAIN, + updated: new Date(Date.now() - (5 * 60_000 - 1)), + action: { system: 'Scrypt', command: 'sell' }, + }); + expect(headroom(justInside)).toBe(1); + expect(justInside.unresolvableTooLong()).toBe(false); + } finally { + jest.useRealTimers(); + } + }); + + it('has passed exactly when the order has become abandonable', () => { + // The two halves of one clock, and they must agree in both directions. The deadline having arrived is what + // lets the reconciliation pass run at all, and being past the bound is what lets it give the order up: a + // state where one holds and the other does not is a pass that runs, re-stamps its cooldown and abandons + // nothing — which is how a five-minute bound came to give up at six. + for (const minutes of [1, 4, 5, 6, 30, 11 * 60, 12 * 60, 13 * 60, 60 * 24]) + for (const command of ['sell', 'buy', 'withdraw']) { + const order = quarantined(minutes, command); + expect(headroom(order) <= 0).toBe(order.unresolvableTooLong()); + } + }); + + it('applies the same bound to the same action as unresolvableTooLong', () => { + // Both read one shared source, so two actions must disagree here exactly where they disagree there: at + // 30 minutes both Scrypt bounds are long spent, while a transfer at a venue that cannot be asked is not. + expect(headroom(quarantined(30, 'sell'))).toBeLessThan(0); + expect(quarantined(30, 'sell').unresolvableTooLong()).toBe(true); + expect(headroom(quarantined(30, 'withdraw'))).toBeLessThan(0); + expect(quarantined(30, 'withdraw').unresolvableTooLong()).toBe(true); + expect(headroom(quarantined(30, 'withdraw', 'Kraken'))).toBeGreaterThan(0); + expect(quarantined(30, 'withdraw', 'Kraken').unresolvableTooLong()).toBe(false); + }); + }); + describe('resolveAsSent / resolveAsNotSent / requestNotSentRelease', () => { it('accepts a release without acting on it: the order keeps blocking', () => { const order = Object.assign(new LiquidityManagementOrder(), { diff --git a/src/subdomains/core/liquidity-management/entities/liquidity-management-order.entity.ts b/src/subdomains/core/liquidity-management/entities/liquidity-management-order.entity.ts index 2c16e613dc..aa8765faf2 100644 --- a/src/subdomains/core/liquidity-management/entities/liquidity-management-order.entity.ts +++ b/src/subdomains/core/liquidity-management/entities/liquidity-management-order.entity.ts @@ -4,7 +4,7 @@ import { IEntity } from 'src/shared/models/entity'; import { Util } from 'src/shared/utils/util'; import { Price, PriceStep } from 'src/subdomains/supporting/pricing/domain/entities/price'; import { Column, Entity, Index, JoinTable, ManyToOne } from 'typeorm'; -import { LiquidityManagementOrderStatus } from '../enums'; +import { LiquidityManagementOrderStatus, LiquidityManagementSystem } from '../enums'; import { OrderFailedException } from '../exceptions/order-failed.exception'; import { OrderNotProcessableException } from '../exceptions/order-not-processable.exception'; import { OrderOutcomeUnknownException } from '../exceptions/order-outcome-unknown.exception'; @@ -19,6 +19,131 @@ import { LiquidityManagementPipeline } from './liquidity-management-pipeline.ent */ const RELEASE_WITHOUT_VENUE_MINUTES = 60; +/** + * How long a quarantined order may go unaccounted for before cleaning it up is worth attempting. + * + * Reaching this bound does not abandon anything by itself — it starts a cancellation. Whether the order is + * then given up depends on the venue settling every reference it claimed, so an order whose integration + * cannot cancel, or whose venue will not answer, stays quarantined past this bound. + * + * The quarantine was built to wait for an operator, on the reasoning that absence at the venue is not proof + * of non-execution. That reasoning is sound but incomplete: it assumed the wait ends. Where nobody checks a + * venue by hand, it does not — the order stays UNCERTAIN forever and takes its rule with it, so the venue + * stops being served at all. A liquidity rule that never runs again is the larger failure, and it is certain, + * while the double execution being guarded against is merely possible. + * + * What carries abandoning is not a conclusion about the order but two things outside it: the venue has + * confirmed that none of its references can execute any more, and a rule replans from the venue's balance + * rather than from the abandoned order. An order that did execute has already moved that balance, so the + * replan sizes itself against what is actually left. That balance is pushed rather than polled, so a fill + * that has only just landed may briefly not be in it — see the adapter's cancellation for that window. + * + * That argument is only as good as the balance behind it, which is why abandoning is confined to orders an + * integration can actually ask the venue about — in practice an exchange, read live at plan time. It is + * deliberately NOT extended to orders no integration can look up: a chain balance omits transactions that + * are sent but unconfirmed, and a bank balance is carried over from the last imported batch, so for those + * the replan could well be sizing itself against a balance the execution has not reached yet. + * + * Within an askable venue, though, an outcome nobody observed at least gets an automatic attempt at cleaning + * itself up, rather than only ever an operator who never comes. Leaving those to a person sounds like the + * careful choice, but where nobody performs the manual release it is not caution, it is a rule that never + * runs again. + * + * "Gets an attempt" is the whole claim, and it is much weaker than "is bounded": the attempt is a cancellation + * (or, for a Scrypt withdrawal that has no cancel, a venue history reply that does not name it), and a + * venue that will not confirm still holds its order here indefinitely. What these bounds end is the assumption + * that somebody will eventually look; the manual release stays a shortcut, not the only way out for venues + * that can answer. + * + * Each bound has to outlast the window in which its order could still be in flight, and that window differs + * by an order of magnitude between kinds of request, so a single value would be either useless or unsafe. + * The two answered bounds are anchored on what completed Scrypt orders actually took over the 30 days to + * 2026-07-29, measured in prod: + * + * trades (n=55): median 9.6s p95 19.8s max 57.1s + * withdrawals (n=49): median 7.7min p95 82min max 5.6h + * + * Read those as a floor, not a ceiling: they describe orders that finished, so one that never becomes + * observable at all is by construction absent from them. Which is why nothing is concluded from the clock + * alone — it only decides when cleaning up is worth attempting. What makes giving up safe is the venue + * confirming that none of the order's references can still execute. + * + * Balances refresh every minute and the pipeline runs every 10 seconds, so no bound is limited by how + * quickly an abandonment can be noticed — only by how long the request itself may still be alive. + */ +const ABANDON_UNCERTAIN_MINUTES = { + /** + * Settled inside the venue, no network leg. Five minutes is roughly five times the slowest such order + * observed, which leaves room for a market phase that keeps one open longer than anything on record. + */ + TRADE: 5, + /** + * A withdrawal at a venue that answers about its own transaction history — today only Scrypt. + * + * Five minutes, and deliberately not derived from how long withdrawals take: measured over 60 days they + * run to 336 minutes, and 39% of them past ten. That tail does not reach this bound, because the bound + * only applies while the venue does NOT name the reference. One it does name is SENT and runs to + * completion on its own clock, but not unbounded — without a transaction hash, + * `SCRYPT_WITHDRAWAL_STUCK_AFTER_MINUTES` (24 hours, in the adapter) fails it so the rule can replan. + * + * What the short value buys is the ceiling the orderer set: five minutes here plus the five a withdrawal + * may stay unobservable (see SCRYPT_UNOBSERVABLE_QUARANTINE_MINUTES) is the ten-minute maximum a + * quarantined order may take to resolve itself. Ten nominal, not to the second: the pass runs on a + * ten-second cron with a few seconds of jitter, and a deadline missed just after a tick waits out the + * one-minute cooldown floor. Read it as ten to eleven minutes plus the venue round-trip. What it costs is + * the case where the venue accepted a withdrawal and publishes it late: it is then given up and reissued. + * That is an internal rebooking — every Scrypt withdrawal address is DFX-owned — and accepted on exactly + * that ground; see `confirmWithdrawalAbsent` for the same trade-off argued at the check itself. + */ + VENUE_WITHDRAWAL: 5, + /** + * Everything else: transfers, withdrawals, bridges, mints. Twice the slowest withdrawal observed, because + * the tail here is genuinely long — a bound near the median would reach orders that are simply still + * running, and reissuing those is what actually moves funds twice. + * + * Reaching this bound only starts the automatic exit attempt — it abandons nothing by itself. For trades + * that attempt is a cancellation whose venue confirmation lets the order go; for a Scrypt withdrawal, + * which has no cancel operation at the venue, it is a confirmed absence from the venue's full transaction + * history. Other transfer kinds and venues that cannot answer that question keep waiting past this bound + * (on the venue, or until an operator releases them as a shortcut). This value is the age at which trying + * is worth it, not a guarantee that giving up is safe. + */ + TRANSFER: 12 * 60, +}; + +/** + * Actions that settle inside a venue rather than moving funds across one, as `system/command` pairs. + * + * Keyed on both halves, because the command name alone does not say what an action does: `sell` on an + * exchange is matched off against a book in seconds, while `sell` on the DEX adapter is an on-chain swap + * with a confirmation time. Matching the name alone would hand the short bound to exactly the actions that + * least deserve it. + * + * An allowlist, not a denylist: anything unrecognised — a new adapter, a renamed command — gets the long + * bound. Being slow to abandon costs a rule some minutes; being fast to abandon a transfer that is still in + * flight is what duplicates it. + * + * The system half comes from the enum so a rename cannot silently detach the list from reality. The command + * half stays literal: those enums live in the adapters, which import this entity, and importing them back + * would close a cycle. + */ +const VENUE_INTERNAL_ACTIONS = [ + `${LiquidityManagementSystem.SCRYPT}/buy`.toLowerCase(), + `${LiquidityManagementSystem.SCRYPT}/sell`.toLowerCase(), +]; + +/** + * Withdrawals at a venue that can be asked about its own transaction history, as `system/command` pairs. + * + * Separate from the general transfer bound on purpose. Lowering TRANSFER itself would also shorten the + * bound for bridges, mints and other systems' transfers — and none of those adapters implements + * `cancelOutstanding`, so no exit would be attempted there anyway; the only effect would be asking their + * venues more often for nothing. + * + * Same allowlist discipline as {@link VENUE_INTERNAL_ACTIONS}: anything unrecognised keeps the long bound. + */ +const VENUE_SELF_RESOLVING_TRANSFERS = [`${LiquidityManagementSystem.SCRYPT}/withdraw`.toLowerCase()]; + @Entity() export class LiquidityManagementOrder extends IEntity { @Column({ length: 256, nullable: false }) @@ -242,6 +367,98 @@ export class LiquidityManagementOrder extends IEntity { return this.notSentRecheckDue != null && Util.minutesDiff(this.notSentRecheckDue) > RELEASE_WITHOUT_VENUE_MINUTES; } + /** + * Whether an unresolved order is old enough that cleaning it up is worth attempting. + * + * Gates both inconclusive outcomes — a venue that answers and has no record, and one that could not be + * asked — because neither improves with waiting. Not a safety judgement on its own: what follows is a + * cancellation, and only the venue confirming that nothing can execute makes giving up safe. See + * {@link ABANDON_UNCERTAIN_MINUTES}. + */ + unresolvableTooLong(): boolean { + // Measured from `updated`, not `created`: an order can run normally for a long time and only become + // UNCERTAIN late, when a completion check amends or restarts it and that write goes unconfirmed. Its + // `created` is then already old, so a bound read from it would expire on the very first pass while the + // fresh replacement request is seconds old and plausibly still arriving — the exact double-send this + // guards against. `updated` moves with the transition into quarantine, so the clock starts there. + // + // When `updated` is missing at runtime (entity `copy()` clears it; some raw loads omit the column) fall + // back to `created` deliberately — not a silent default. `created` is always older than or equal to + // `updated`, so the bound expires earlier rather than later. That is safe here: automatic abandon still + // requires venue confirmation (cancel settled / confirmed absence); this clock only decides *when* a + // cleanup attempt is worth making, not whether giving up is safe. Preferring `updated` when present + // remains correct for the long-running-then-quarantined case above. + // + // Without either timestamp there is no clock to run out. Guarded explicitly because Util.minutesDiff + // treats a missing date as the epoch and would report tens of millions of minutes — abandoning instantly + // every order whose dates were not loaded. Absent evidence this must hold the order, never drop it. + const since = this.updated ?? this.created; + if (!since) return false; + + // `>=`, so that reaching the bound is enough. With `>`, the instant the bound is exactly met left the two + // halves of this clock disagreeing: {@link getAbandonableAt} already reported nothing left — which is + // what lets the reconciliation pass run at that moment — while this said not yet. The pass then re-stamped + // its cooldown and the order waited out another full interval, so a five-minute bound gave up at six. + return Util.minutesDiff(since) >= this.getAbandonBoundMinutes(); + } + + /** + * The moment this order becomes abandonable — null while it never does. + * + * The same bound and the same clock as {@link unresolvableTooLong}, stated as an absolute instant so that a + * caller deciding *when to look at this order again* can keep its own wait on the near side of it. A pass + * throttled past this point would push the abandonment beyond the ceiling that bound exists to impose, and + * nothing in the throttle itself would reveal that it had. + * + * Absolute rather than "time remaining" on purpose. A remaining duration has to be clamped at zero, and that + * clamp erases the one fact a throttle needs after the deadline: whether the wait it is about to impose + * started before it. Measured against a fixed instant the question does not arise — and the caller can ask + * about any moment, not only about now. + * + * Null without `updated` and without `created`, mirroring that method's refusal to run a clock it does not + * have: no deadline to respect means no constraint to impose on a caller's wait. + */ + getAbandonableAt(): Date | null { + // Same clock as {@link unresolvableTooLong}: prefer `updated`, fall back to `created` when `updated` was + // not loaded (see that method for why the fallback is earlier-not-later and still safe). Both missing → + // no deadline, mirroring the refusal to run a clock we do not have. + const since = this.updated ?? this.created; + if (!since) return null; + + return new Date(since.getTime() + this.getAbandonBoundMinutes() * 60_000); + } + + /** + * The quarantine bound that applies to this order's action, in minutes. Deliberately the single place that + * reads {@link ABANDON_UNCERTAIN_MINUTES}: the deadline and the remaining time to it must never be able to + * disagree about which bound they are talking about. + */ + private getAbandonBoundMinutes(): number { + // Unknown, unloaded or unlisted action falls to the long bound, for the same reason as a missing date. + const action = `${this.action?.system}/${this.action?.command}`.toLowerCase(); + + if (VENUE_INTERNAL_ACTIONS.includes(action)) return ABANDON_UNCERTAIN_MINUTES.TRADE; + if (VENUE_SELF_RESOLVING_TRANSFERS.includes(action)) return ABANDON_UNCERTAIN_MINUTES.VENUE_WITHDRAWAL; + + return ABANDON_UNCERTAIN_MINUTES.TRANSFER; + } + + /** + * End a quarantined order that has nothing left outstanding at the venue, and let the rule move on. + * + * FAILED rather than a verified non-execution: nothing here establishes that the request never took + * effect — a reference may well have executed — and the reason says so, so the record does not claim more + * than was actually observed. Named for the state it ends rather than for what ended it, because several + * routes arrive here. + */ + abandonUncertain(reason: string): this { + this.status = LiquidityManagementOrderStatus.FAILED; + this.errorMessage = reason; + this.notSentRecheckDue = null; + + return this; + } + /** * Accept somebody's judgement that this order never left — without acting on it yet. * diff --git a/src/subdomains/core/liquidity-management/enums/index.ts b/src/subdomains/core/liquidity-management/enums/index.ts index edddb69185..6595e46e5d 100644 --- a/src/subdomains/core/liquidity-management/enums/index.ts +++ b/src/subdomains/core/liquidity-management/enums/index.ts @@ -42,6 +42,17 @@ export enum LiquidityManagementOrderStatus { // Quarantine for an order whose request left our side without an observed outcome. Terminal for the // pipeline (it never resumes on its own) but not for the order: `resolveUncertainOrders` asks the venue // what happened and moves it on to IN_PROGRESS or FAILED. See OrderOutcomeUnknownException. + // + // Where an integration can ask the venue, an automatic cleanup is at least attempted: once the order has + // outlived the window in which its request could still be in flight (ABANDON_UNCERTAIN_MINUTES, which + // differs for venue-internal trades and transfers), the adapter is asked to cancel every reference it + // claimed — if it supports cancelling that kind of request at all. + // + // Giving up is never concluded from the clock alone: past the bound the venue is asked to cancel every + // reference the order claimed — sent or merely reserved — and only its answer that none can still execute + // permits FAILED. + // A venue that will not settle them, and an adapter that cannot cancel at all, keep waiting — for + // `resolveUncertainOrderManually`. UNCERTAIN = 'Uncertain', } @@ -51,14 +62,21 @@ export enum UncertainOrderResolution { SENT = 'Sent', /** The venue demonstrably does not know the order — nothing was executed, the rule may plan anew. */ NOT_SENT = 'NotSent', - /** The venue answered, and the answer settles nothing. Stay in quarantine and look again later. */ + /** + * The venue answered, and the answer settles nothing. Stay in quarantine and look again later — until the + * order outlives the abandon bound for its kind of request, at which point the caller tries to cancel + * everything it sent. Only that confirmation releases it; the bound alone never does. + */ UNRESOLVED = 'Unresolved', /** - * The venue could not be asked at all. + * The venue could not be asked, or could not be asked completely. * * Deliberately not the same as UNRESOLVED: that one is an answer, this one is the absence of one, and a * caller that retires an order's outstanding work on the strength of a completed lookup must not retire it * on a failed one. + * + * "Not completely" covers an order with no reference to ask about at all: there is nothing to look up, so + * nothing was learned. */ UNAVAILABLE = 'Unavailable', } diff --git a/src/subdomains/core/liquidity-management/factories/liquidity-action-integration.factory.ts b/src/subdomains/core/liquidity-management/factories/liquidity-action-integration.factory.ts index cd7b6587a8..08ea6beb21 100644 --- a/src/subdomains/core/liquidity-management/factories/liquidity-action-integration.factory.ts +++ b/src/subdomains/core/liquidity-management/factories/liquidity-action-integration.factory.ts @@ -71,4 +71,17 @@ export class LiquidityActionIntegrationFactory { return null; } + + /** + * Resolve the adapter that can talk to the venue for a quarantined order. + * + * Unlike {@link getIntegration}, this ignores `supportedCommands`: reconciling an already-quarantined + * order cares who can ask the venue, not whether the command is still registered as an executable action. + * A command rename or removal must not leave UNCERTAIN rows stranded for an operator forever — the + * adapter for the system still knows how to look up and settle references. {@link getIntegration} stays + * the gate for starting new work; only a registered command may execute. + */ + getReconciliationIntegration(action: LiquidityManagementAction): LiquidityActionIntegration { + return this.adapters.get(action.system) ?? null; + } } diff --git a/src/subdomains/core/liquidity-management/interfaces/index.ts b/src/subdomains/core/liquidity-management/interfaces/index.ts index fed3e41c52..f394437e55 100644 --- a/src/subdomains/core/liquidity-management/interfaces/index.ts +++ b/src/subdomains/core/liquidity-management/interfaces/index.ts @@ -33,6 +33,36 @@ export interface LiquidityActionIntegration { * quarantine for a human to resolve. */ resolveUncertainOrder?(order: LiquidityManagementOrder): Promise; + + /** + * Make sure nothing this order has claimed — sent or merely reserved — can still execute, so it may be + * given up safely. + * + * The one thing that makes abandoning a quarantined order dangerous is a request still live at the venue: + * give the rule its funds back and a late fill spends them twice. Rather than estimating when that can no + * longer happen, this removes the possibility — cancelling is the opposite of re-sending, so it is the one + * write that is always safe against an outcome nobody could observe. Where the venue has no cancel for the + * request kind (a Scrypt withdrawal), the substitute is weaker and knowingly so: the venue answered and did + * not name the reference, with no completeness check on that answer. + * + * Returns a non-empty reason string only when the venue has answered that nothing under this order is left + * to execute (or, for a withdrawal, that a successful history reply does not name it and it did not surface + * in the live cache meanwhile). The caller writes that + * string into the order and the log as-is — each integration supplies its own wording so the pipeline never + * invents a reason the venue never gave. `null` means no automatic exit; the order stays quarantined. + * Read "answered" precisely: a cancellation it accepts, an order it reports terminal, or a successful + * history reply that omits the withdrawal reference settles the question. An unconfirmed cancel must return + * null: it may well have taken effect, but "may well" is what quarantine already means. A truncated history + * is deliberately NOT rejected for withdrawals — that trade-off, and why the alternative was worse, is + * argued where the check lives. + * Integrations that cannot cancel omit this and simply have no automatic exit from quarantine. For Scrypt, + * reconciliation reaches the adapter by system (not by registered command), so every command — including + * one no longer in `supportedCommands` — gets either a venue-confirmed reason string or `null`. Known + * trade/withdraw commands cancel or confirm absence as before; an unsupported command asks `getOrderStatus` + * per reference and cancels any non-terminal one under the symbol that reply itself carries — so "no + * derivable symbol" is not a reason to wait. Neither path waits on an operator as its way out. + */ + cancelOutstanding?(order: LiquidityManagementOrder): Promise; } export interface LiquidityState { diff --git a/src/subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.spec.ts b/src/subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.spec.ts index 3e835685a0..571dc66537 100644 --- a/src/subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.spec.ts +++ b/src/subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.spec.ts @@ -1,5 +1,6 @@ import { createMock } from '@golevelup/ts-jest'; import { NotificationService } from 'src/subdomains/supporting/notification/services/notification.service'; +import { FindOperator } from 'typeorm'; import { LiquidityManagementOrder } from '../entities/liquidity-management-order.entity'; import { LiquidityManagementPipeline } from '../entities/liquidity-management-pipeline.entity'; import { LiquidityManagementRule } from '../entities/liquidity-management-rule.entity'; @@ -12,6 +13,7 @@ import { } from '../enums'; import { OrderOutcomeUnknownException } from '../exceptions/order-outcome-unknown.exception'; import { LiquidityActionIntegrationFactory } from '../factories/liquidity-action-integration.factory'; +import { LiquidityActionIntegration } from '../interfaces'; import { LiquidityManagementOrderRepository } from '../repositories/liquidity-management-order.repository'; import { LiquidityManagementPipelineRepository } from '../repositories/liquidity-management-pipeline.repository'; import { LiquidityManagementRuleRepository } from '../repositories/liquidity-management-rule.repository'; @@ -169,6 +171,7 @@ describe('LiquidityManagementPipelineService', () => { correlationId: 'dfx-lm-9', errorMessage: 'Scrypt did not answer', created: ORDER_CREATED, + updated: new Date(Date.now() - 60_000), action: { id: 233, system: 'Scrypt', command: 'sell' }, ...overrides, }); @@ -184,14 +187,30 @@ describe('LiquidityManagementPipelineService', () => { }); } - function stubIntegration(resolution: UncertainOrderResolution): void { - jest.spyOn(actionIntegrationFactory, 'getIntegration').mockReturnValue({ + /** `cancelSettles` is what the venue says when asked to make sure nothing can execute any more. + * true → reason string (exit), false → null (no exit). The pipeline now receives the reason from the + * integration rather than inventing one. */ + function stubIntegration(resolution: UncertainOrderResolution, cancelSettles = true): LiquidityActionIntegration { + // resolveUncertainOrders looks up the venue via getReconciliationIntegration (system-only). The SENT + // path additionally consults getIntegration: only a registered command may return to IN_PROGRESS. + // Stub both with the same adapter so existing tests model the normal registered-command case they + // always intended (command: 'sell' / supportedCommands: ['sell']). + const integration = { supportedCommands: ['sell'], executeOrder: jest.fn(), checkCompletion: jest.fn(), validateParams: jest.fn(), resolveUncertainOrder: jest.fn().mockResolvedValue(resolution), - }); + cancelOutstanding: jest + .fn() + .mockResolvedValue( + cancelSettles ? 'the venue answered for every reference that nothing is left to execute' : null, + ), + }; + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue(integration); + jest.spyOn(actionIntegrationFactory, 'getIntegration').mockReturnValue(integration); + + return integration; } it('only ever asks about quarantined orders', async () => { @@ -202,6 +221,8 @@ describe('LiquidityManagementPipelineService', () => { expect(findBy).toHaveBeenCalledWith({ status: LiquidityManagementOrderStatus.UNCERTAIN }); }); + // SENT → IN_PROGRESS covers FIX 1 normal case (command still registered via stubIntegration's + // getIntegration mock). The unregistered-command SENT case is asserted separately below. it.each([ [UncertainOrderResolution.SENT, LiquidityManagementOrderStatus.IN_PROGRESS], [UncertainOrderResolution.NOT_SENT, LiquidityManagementOrderStatus.FAILED], @@ -218,6 +239,258 @@ describe('LiquidityManagementPipelineService', () => { expect(order.status).toBe(expectedStatus); }); + /** Quarantined `minutes` ago — the clock runs from `updated`, not from creation. */ + function agedOrder(minutes: number, command = 'sell', system = 'Scrypt'): LiquidityManagementOrder { + return uncertainOrder({ + created: new Date(Date.now() - 30 * 24 * 60 * 60 * 1000), + updated: new Date(Date.now() - minutes * 60 * 1000), + action: { id: 233, system, command } as LiquidityManagementOrder['action'], + }); + } + + function expectResolution( + order: LiquidityManagementOrder, + resolution: UncertainOrderResolution, + ): LiquidityActionIntegration { + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + jest.spyOn(orderRepo, 'update').mockResolvedValue({ affected: 1, raw: [], generatedMaps: [] }); + return stubIntegration(resolution); + } + + // both allowlist entries, so dropping or mistyping either one is caught + it.each(['sell', 'buy'])( + 'abandons a %s past its bound once the venue settles every reference, so its rule is not blocked forever', + async (command) => { + // the failure this prevents: nobody releases the order by hand, so it stays UNCERTAIN indefinitely + // and the rule behind it never plans again — the venue silently stops being served + const order = agedOrder(30, command); + expectResolution(order, UncertainOrderResolution.UNRESOLVED); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.FAILED); + // the record must not claim an observation nobody made + expect(order.errorMessage).toContain('answered for every reference'); + }, + ); + + it('acts on a release immediately when the order carries no reference to look up', async () => { + // an unaskable order reports UNAVAILABLE, so without this the release would sit out the full + // unreachable-venue wait for an answer that can never arrive — somebody already checked by hand + const order = uncertainOrder({ correlationId: undefined, notSentRecheckDue: RELEASED_AT }); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + jest.spyOn(orderRepo, 'update').mockResolvedValue({ affected: 1, raw: [], generatedMaps: [] }); + stubIntegration(UncertainOrderResolution.UNAVAILABLE); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.FAILED); + expect(order.errorMessage).toContain('no reference exists to look it up'); + }); + + it('lets a human release win over the clock when both would apply', async () => { + // Two exits apply here and they record different verdicts: the release says the venue confirmed the + // request never arrived, the abandon says only that nothing is left to execute. The operator's own + // reason survives either way — both prefix the existing message rather than replacing it — but which + // verdict is added to it matters, and only the release rests on somebody having actually checked. The + // branch order decides that, so it is asserted here: reordering the chain later must not quietly file + // an audited case under the weaker of the two. + const order = agedOrder(30); + order.notSentRecheckDue = RELEASED_AT; + order.errorMessage = 'Scrypt did not answer (released by account 42: venue checked — ticket OPS-42)'; + expectResolution(order, UncertainOrderResolution.UNRESOLVED); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.FAILED); + expect(order.errorMessage).toContain('the venue has no record of it either'); + expect(order.errorMessage).not.toContain('abandoned'); + }); + + it('does not abandon while the venue will not settle its references', async () => { + // the only thing that makes giving up dangerous is a request that can still execute. If the venue + // will not confirm that none can — unreachable, or an order it reports in another state — then the + // order keeps waiting. Nothing is concluded from that silence, which is the point. + const order = agedOrder(30); + expectResolution(order, UncertainOrderResolution.UNRESOLVED); + stubIntegration(UncertainOrderResolution.UNRESOLVED, false); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + }); + + it('abandons an order whose references the venue settled, whatever the lookup said', async () => { + // the cancel is what settles it, so an inconclusive lookup is no obstacle: once nothing can execute, + // giving up is a fact rather than an estimate + const order = agedOrder(30); + expectResolution(order, UncertainOrderResolution.UNAVAILABLE); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.FAILED); + }); + + it('keeps a trade quarantined inside its bound — the slowest observed trade took under a minute', async () => { + const order = agedOrder(1); + expectResolution(order, UncertainOrderResolution.UNRESOLVED); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + }); + + it('holds a transfer nobody can ask about far longer than a Scrypt one', async () => { + // Scrypt withdrawals now share the short bound: the venue answers about its own history, and the + // ten-minute ceiling outweighs the long tail, since a reissued payout only moves funds between our own + // accounts. A bridge or mint has neither property, so the long bound stays where it is. + const order = agedOrder(30, 'withdraw', 'Kraken'); + expectResolution(order, UncertainOrderResolution.UNRESOLVED); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + }); + + it('abandons a Scrypt withdrawal past its own five-minute bound, not the transfer one', async () => { + // the failure this prevents: a typo or case mismatch in the allowlist that maps Scrypt withdrawals + // to their five-minute bound would leave them on the long transfer bound unnoticed. + const order = agedOrder(6, 'withdraw'); + const integration = expectResolution(order, UncertainOrderResolution.UNRESOLVED); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.FAILED); + expect(integration.cancelOutstanding).toHaveBeenCalledWith(order); + }); + + it('keeps a Scrypt withdrawal quarantined while it is still inside its own five-minute bound', async () => { + // the failure this prevents: cancelOutstanding would be called and the withdrawal abandoned before + // its own five-minute bound has run out. + const order = agedOrder(4, 'withdraw'); + const integration = expectResolution(order, UncertainOrderResolution.UNRESOLVED); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + expect(integration.cancelOutstanding).not.toHaveBeenCalled(); + }); + + it('abandons a transfer once even its long bound has run out and the venue settles it', async () => { + // the bound alone is not enough: this asserts the pipeline's side of the contract, that an aged + // transfer whose integration returns a reason string from cancelOutstanding does get abandoned. + // How the integration settles the question (trade cancel vs. a withdrawal history reply that does not name it) + // is its business — the pipeline only forwards the returned reason. + const order = agedOrder(13 * 60, 'withdraw'); + expectResolution(order, UncertainOrderResolution.UNRESOLVED); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.FAILED); + expect(order.errorMessage).toContain('answered for every reference'); + }); + + it('gives an unrecognised command the long bound, not the short one', async () => { + // allowlist, not denylist: a new adapter must not inherit the trade bound by accident + const order = agedOrder(30, 'some-new-bridge-command'); + expectResolution(order, UncertainOrderResolution.UNRESOLVED); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + }); + + it('never abandons an order no integration can look up, however old', async () => { + // the venue is never asked here, so only the clock would be left — and the safety of abandoning rests + // on replanning against a balance that reflects an execution. A chain balance omits unconfirmed + // transactions and a bank balance comes from the last import, so that does not hold off-exchange. + const order = agedOrder(30 * 24 * 60); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + jest.spyOn(orderRepo, 'update').mockResolvedValue({ affected: 1, raw: [], generatedMaps: [] }); + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue(undefined); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + }); + + it('gives an on-chain swap the long bound even though its command is called "sell"', async () => { + // the command name alone does not say what an action does: DfxDex/sell is an on-chain swap with a + // confirmation time, not a book match settled in seconds + const order = agedOrder(30, 'sell', 'DfxDex'); + expectResolution(order, UncertainOrderResolution.UNRESOLVED); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + }); + + it('narrows an abandon on the absent release as SQL NULL, not as a bare null', async () => { + // the abandon concludes nothing about the send itself, so it must not outrank an operator who checked and is + // still owed one venue answer — hence the narrowing. But "no release pending" is the ordinary case + // here, and a raw null renders as `= NULL`, which matches no row at all: the update would never + // affect anything and would report itself as a lost race. A mocked repo cannot see that, so assert + // on the operator itself. + const order = agedOrder(30); + const update = jest.spyOn(orderRepo, 'update').mockResolvedValue({ affected: 1, raw: [], generatedMaps: [] }); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + stubIntegration(UncertainOrderResolution.UNRESOLVED); + + await service['resolveUncertainOrders'](); + + const criteria = update.mock.calls[0][0] as { notSentRecheckDue: FindOperator }; + expect(criteria.notSentRecheckDue).toBeInstanceOf(FindOperator); + expect(criteria.notSentRecheckDue.type).toBe('isNull'); + expect(order.status).toBe(LiquidityManagementOrderStatus.FAILED); + }); + + it('narrows a not-sent verdict on the absent release as SQL NULL too', async () => { + // the same trap on the other caller: a venue verdict arrives with nobody having released the order, + // so its examined value is null as well. Asserting on status alone would not catch a regression here, + // because resolveAsNotSent already sets FAILED synchronously before the write is attempted. + const order = uncertainOrder(); + const update = jest.spyOn(orderRepo, 'update').mockResolvedValue({ affected: 1, raw: [], generatedMaps: [] }); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + stubIntegration(UncertainOrderResolution.NOT_SENT); + + await service['resolveUncertainOrders'](); + + const criteria = update.mock.calls[0][0] as { notSentRecheckDue: FindOperator }; + expect(criteria.notSentRecheckDue).toBeInstanceOf(FindOperator); + expect(criteria.notSentRecheckDue.type).toBe('isNull'); + }); + + it('runs the abandon clock off `created` once `updated` is missing, because created is always the older (or equal) bound', async () => { + // entity copy() clears `updated`, and some raw loads omit the column — `created` stays the only + // clock left. It is never younger than `updated` would have been, so falling back to it can only make + // the bound expire earlier, never later: the one direction that keeps giving up safe. + const order = uncertainOrder({ updated: undefined }); + expectResolution(order, UncertainOrderResolution.UNRESOLVED); + + await service['resolveUncertainOrders'](); + + // ORDER_CREATED is 29 days old, decisively past the 5-minute trade bound + expect(order.status).toBe(LiquidityManagementOrderStatus.FAILED); + }); + + it('never abandons an order with neither a quarantine timestamp nor a creation date', async () => { + // Util.minutesDiff reads a missing date as the epoch; unguarded that abandons instantly. `created` is + // the last fallback, and losing that too must leave no clock at all — the order can only wait. + // + // Routed through a pending release (rather than a bare uncertainOrder()) so the pass takes the + // releasePending branch: that branch never reads order.created for the cooldown throttle, so this + // reaches unresolvableTooLong()'s own missing-date guard instead of the unrelated crash a bare + // uncertain order with no `created` would hit in the age-based throttle a few lines above it. + const order = uncertainOrder({ updated: undefined, created: undefined, notSentRecheckDue: RELEASED_AT }); + stubIntegration(UncertainOrderResolution.UNAVAILABLE); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + expect(orderRepo.update).not.toHaveBeenCalled(); + }); + it('keeps a released order quarantined until the venue has actually answered', async () => { // the release is a judgement, and while it is unconfirmed the order must not become terminal — // a terminal order lets its rule plan again against funds that may well be committed @@ -266,6 +539,23 @@ describe('LiquidityManagementPipelineService', () => { expect(update.mock.calls[0][0]).toMatchObject({ notSentRecheckDue: RELEASED_AT }); }); + it('does not report a missed quarantine write as a resolution that happened elsewhere', async () => { + // The write cannot tell a race from a narrowing that will never match again, and the second case repeats + // forever while the order stays as it is. Claiming the benign one hid exactly that: a release timestamp + // with microsecond precision is unmatchable by a JS Date, and it held a live order for hours behind the + // reassuring wording. So the line must name both and be a warning, not routine information. + const order = releasePendingOrder(); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + jest.spyOn(orderRepo, 'update').mockResolvedValue({ affected: 0, raw: [], generatedMaps: [] }); + const warn = jest.spyOn(service['logger'], 'warn').mockImplementation(() => undefined); + stubIntegration(UncertainOrderResolution.UNRESOLVED); + + await service['resolveUncertainOrders'](); + + expect(warn).toHaveBeenCalledWith(expect.stringMatching(/was not updated.*resolved elsewhere.*matched no row/s)); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('stuck')); + }); + it('does not release an unreleased order on the same inconclusive answer', async () => { // absence is not proof; without somebody having checked independently there is only one negative const order = uncertainOrder(); @@ -308,7 +598,7 @@ describe('LiquidityManagementPipelineService', () => { const order = releasePendingOrder(); jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); const update = jest.spyOn(orderRepo, 'update').mockResolvedValue({ affected: 1, raw: [], generatedMaps: [] }); - jest.spyOn(actionIntegrationFactory, 'getIntegration').mockReturnValue(null); + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue(null); // and the pass reports the change, so the caller's loop knows something moved await expect(service['resolveUncertainOrders']()).resolves.toBe(true); @@ -322,7 +612,7 @@ describe('LiquidityManagementPipelineService', () => { it('leaves an unreleased order alone when its adapter is gone', async () => { const order = uncertainOrder(); jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); - jest.spyOn(actionIntegrationFactory, 'getIntegration').mockReturnValue(null); + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue(null); await service['resolveUncertainOrders'](); @@ -470,7 +760,7 @@ describe('LiquidityManagementPipelineService', () => { it('keeps the order quarantined when the lookup itself throws', async () => { const order = uncertainOrder(); jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); - jest.spyOn(actionIntegrationFactory, 'getIntegration').mockReturnValue({ + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue({ supportedCommands: ['sell'], executeOrder: jest.fn(), checkCompletion: jest.fn(), @@ -483,6 +773,150 @@ describe('LiquidityManagementPipelineService', () => { expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); }); + it('reaches the adapter for a Scrypt order whose command is no longer registered', async () => { + // getIntegration would return null for an unregistered command; reconciliation resolves by system + const resolveUncertainOrder = jest.fn().mockResolvedValue(UncertainOrderResolution.UNRESOLVED); + const cancelOutstanding = jest.fn().mockResolvedValue(null); + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue({ + supportedCommands: ['sell', 'buy', 'withdraw'], // deliberately omits the order's command + executeOrder: jest.fn(), + checkCompletion: jest.fn(), + validateParams: jest.fn(), + resolveUncertainOrder, + cancelOutstanding, + }); + // getIntegration would skip — prove reconciliation does not use it for this path + jest.spyOn(actionIntegrationFactory, 'getIntegration').mockReturnValue(null); + const order = agedOrder(30, 'sell-if-deficit', 'Scrypt'); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + jest.spyOn(orderRepo, 'update').mockResolvedValue({ affected: 1, raw: [], generatedMaps: [] }); + + await service['resolveUncertainOrders'](); + + expect(resolveUncertainOrder).toHaveBeenCalledWith(order); + }); + + it('keeps a venue-confirmed SENT order quarantined when its command is no longer registered', async () => { + // Venue knows the reference, but no registered command can checkCompletion. Returning to IN_PROGRESS + // would trap the order (see FIX 1); stay UNCERTAIN so the automatic abandon path remains available. + const resolveUncertainOrder = jest.fn().mockResolvedValue(UncertainOrderResolution.SENT); + const cancelOutstanding = jest.fn().mockResolvedValue(null); + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue({ + supportedCommands: ['sell', 'buy', 'withdraw'], // deliberately omits the order's command + executeOrder: jest.fn(), + checkCompletion: jest.fn(), + validateParams: jest.fn(), + resolveUncertainOrder, + cancelOutstanding, + }); + jest.spyOn(actionIntegrationFactory, 'getIntegration').mockReturnValue(null); + const order = agedOrder(30, 'sell-if-deficit', 'Scrypt'); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + const warn = jest.spyOn(service['logger'], 'warn').mockImplementation(() => undefined); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + expect(order.status).not.toBe(LiquidityManagementOrderStatus.IN_PROGRESS); + expect(resolveUncertainOrder).toHaveBeenCalledWith(order); + expect(warn).toHaveBeenCalledWith(expect.stringMatching(/stays quarantined.*no registered command/s)); + expect(cancelOutstanding).not.toHaveBeenCalled(); + }); + + it('abandons a venue-confirmed SENT order past its bound when its command is no longer registered', async () => { + const resolveUncertainOrder = jest.fn().mockResolvedValue(UncertainOrderResolution.SENT); + const cancelOutstanding = jest + .fn() + .mockResolvedValue('the venue answered for every reference that nothing is left to execute'); + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue({ + supportedCommands: ['sell', 'buy', 'withdraw'], // deliberately omits the order's command + executeOrder: jest.fn(), + checkCompletion: jest.fn(), + validateParams: jest.fn(), + resolveUncertainOrder, + cancelOutstanding, + }); + jest.spyOn(actionIntegrationFactory, 'getIntegration').mockReturnValue(null); + const order = agedOrder(30, 'sell', 'Scrypt'); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + jest.spyOn(orderRepo, 'update').mockResolvedValue({ affected: 1, raw: [], generatedMaps: [] }); + jest.spyOn(service['logger'], 'warn').mockImplementation(() => undefined); + + await service['resolveUncertainOrders'](); + + expect(cancelOutstanding).toHaveBeenCalledWith(order); + expect(order.status).toBe(LiquidityManagementOrderStatus.FAILED); + expect(order.errorMessage).toContain('the venue answered for every reference that nothing is left to execute'); + }); + + it('keeps a venue-confirmed SENT order past its bound quarantined when cancelOutstanding is unsettled', async () => { + const resolveUncertainOrder = jest.fn().mockResolvedValue(UncertainOrderResolution.SENT); + const cancelOutstanding = jest.fn().mockResolvedValue(null); + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue({ + supportedCommands: ['sell', 'buy', 'withdraw'], // deliberately omits the order's command + executeOrder: jest.fn(), + checkCompletion: jest.fn(), + validateParams: jest.fn(), + resolveUncertainOrder, + cancelOutstanding, + }); + jest.spyOn(actionIntegrationFactory, 'getIntegration').mockReturnValue(null); + const order = agedOrder(30, 'sell', 'Scrypt'); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + jest.spyOn(service['logger'], 'warn').mockImplementation(() => undefined); + + await service['resolveUncertainOrders'](); + + expect(cancelOutstanding).toHaveBeenCalledWith(order); + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + }); + + it('abandons a released venue-confirmed SENT order past its bound via cancelOutstanding, not completeNotSentRelease', async () => { + const resolveUncertainOrder = jest.fn().mockResolvedValue(UncertainOrderResolution.SENT); + const cancelOutstanding = jest + .fn() + .mockResolvedValue('the venue answered for every reference that nothing is left to execute'); + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue({ + supportedCommands: ['sell', 'buy', 'withdraw'], // deliberately omits the order's command + executeOrder: jest.fn(), + checkCompletion: jest.fn(), + validateParams: jest.fn(), + resolveUncertainOrder, + cancelOutstanding, + }); + jest.spyOn(actionIntegrationFactory, 'getIntegration').mockReturnValue(null); + const order = agedOrder(30, 'sell', 'Scrypt'); + order.notSentRecheckDue = RELEASED_AT; + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + jest.spyOn(orderRepo, 'update').mockResolvedValue({ affected: 1, raw: [], generatedMaps: [] }); + jest.spyOn(service['logger'], 'warn').mockImplementation(() => undefined); + + await service['resolveUncertainOrders'](); + + expect(cancelOutstanding).toHaveBeenCalledWith(order); + expect(order.status).toBe(LiquidityManagementOrderStatus.FAILED); + expect(order.errorMessage).toContain('the venue answered for every reference that nothing is left to execute'); + expect(order.errorMessage).not.toContain('the venue confirmed the request never arrived'); + }); + + it('leaves a non-Scrypt system without resolveUncertainOrder alone (no automatic progress)', async () => { + // observable behaviour unchanged vs getIntegration: adapter exists but offers no reconciliation + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue({ + supportedCommands: ['transfer'], + executeOrder: jest.fn(), + checkCompletion: jest.fn(), + validateParams: jest.fn(), + // no resolveUncertainOrder + }); + const order = agedOrder(30 * 24 * 60, 'transfer', 'SomeBank'); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + expect(orderRepo.update).not.toHaveBeenCalled(); + }); + describe('venue-lookup cooldown', () => { // The cooldown is a pure function of Date.now(), so these tests drive the clock instead of waiting. // Scoped here rather than suite-wide: nothing else in this file cares about time. @@ -494,7 +928,7 @@ describe('LiquidityManagementPipelineService', () => { * and only the cooldown decides whether the venue is asked. */ function stubResolver(): jest.Mock { const resolveUncertainOrder = jest.fn().mockResolvedValue(UncertainOrderResolution.UNAVAILABLE); - jest.spyOn(actionIntegrationFactory, 'getIntegration').mockReturnValue({ + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue({ supportedCommands: ['sell'], executeOrder: jest.fn(), checkCompletion: jest.fn(), @@ -615,7 +1049,12 @@ describe('LiquidityManagementPipelineService', () => { // 11 min 20 s is past it. A one-sided assertion would let the rate drift unnoticed: with `ageMs / 5` // the order simply stays in cooldown and a lower-bound-only test keeps passing. const resolveUncertainOrder = stubResolver(); - const order = uncertainOrder({ created: new Date(Date.now() - 100 * 60_000) }); + const order = uncertainOrder({ + created: new Date(Date.now() - 100 * 60_000), + // A venue that cannot be asked keeps the twelve-hour bound, so no deadline tightens the interval + // and the age formula alone is under test here. + action: { id: 233, system: 'Kraken', command: 'withdraw' } as LiquidityManagementOrder['action'], + }); jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); await service['resolveUncertainOrders'](); @@ -629,13 +1068,119 @@ describe('LiquidityManagementPipelineService', () => { expect(resolveUncertainOrder).toHaveBeenCalledTimes(2); }); + it('holds the deadline cap still between lookups instead of letting it shrink', async () => { + // The cap is the time that was left when the last lookup finished. Read fresh on every tick it would + // shrink while the wait grows, the two would meet halfway, and a five-minute bound would be re-asked + // at 2.5 minutes — then 3.75, then 4.4, a geometric series of expensive lookups before a deadline that + // never moved. Asserted just before the bound, where the shrinking variant asks and this one does not. + const order = agedOrder(0); + const resolveUncertainOrder = jest.fn().mockResolvedValue(UncertainOrderResolution.UNAVAILABLE); + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue({ + supportedCommands: ['sell'], + executeOrder: jest.fn(), + checkCompletion: jest.fn(), + validateParams: jest.fn(), + resolveUncertainOrder, + }); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + + await service['resolveUncertainOrders'](); + expect(resolveUncertainOrder).toHaveBeenCalledTimes(1); + + // half the bound: exactly where a cap re-read on this tick would coincide with the elapsed wait + jest.advanceTimersByTime(2.5 * 60_000); + await service['resolveUncertainOrders'](); + expect(resolveUncertainOrder).toHaveBeenCalledTimes(1); + + // one millisecond short of the bound — still nothing to give up, so still nothing to ask + jest.advanceTimersByTime(2.5 * 60_000 - 1); + await service['resolveUncertainOrders'](); + expect(resolveUncertainOrder).toHaveBeenCalledTimes(1); + + // and exactly at the bound it asks, which is what the cap exists to guarantee + jest.advanceTimersByTime(1); + await service['resolveUncertainOrders'](); + expect(resolveUncertainOrder).toHaveBeenCalledTimes(2); + }); + + it('does not let its own floor push the abandonment past the deadline', async () => { + // A lookup that runs shortly before the bound has less than a floor's worth of time left. Raising the cap + // to the floor there schedules the next pass after the deadline — the overshoot the cap exists to prevent, + // caused by the cap. Reached the way it happens in practice: the order enters this pass already close to + // its bound, with no cooldown recorded, so the first lookup lands 30 seconds short of it. + const order = agedOrder(4.5); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + jest.spyOn(orderRepo, 'update').mockResolvedValue({ affected: 1, raw: [], generatedMaps: [] }); + stubIntegration(UncertainOrderResolution.UNRESOLVED); + + // first lookup at 4:30, which stamps the cooldown with only 30 seconds of headroom left + await service['resolveUncertainOrders'](); + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + + // at the bound the order must be gone — a floor measured from 4:30 would hold it until 5:30 + jest.advanceTimersByTime(30_000); + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.FAILED); + }); + + it('never throttles past the bound at which the order becomes abandonable', async () => { + // Where the cooldown and the abandon bound meet. This same pass is what gives an expired order up, so + // a wait longer than what is left of its bound postpones the abandonment — an order weeks old draws + // the full thirty-minute interval, six times a trade's own five-minute bound, and the ceiling this + // branch exists to impose would have been raised with nothing saying so. + const order = agedOrder(0); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + jest.spyOn(orderRepo, 'update').mockResolvedValue({ affected: 1, raw: [], generatedMaps: [] }); + stubIntegration(UncertainOrderResolution.UNRESOLVED); + + // inside the bound: nothing to give up yet + await service['resolveUncertainOrders'](); + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + + // past the bound, still far short of the thirty-minute cap that would otherwise still be running + jest.advanceTimersByTime(6 * 60_000); + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.FAILED); + }); + + it("leaves the cap governing when the order's own bound is further off than the cap", async () => { + // The deadline only ever tightens the interval, never loosens it: a transfer has twelve hours, so the + // cap decides exactly as it did before, and a lookup one millisecond early still must not happen. + const order = agedOrder(0, 'withdraw', 'Kraken'); + const resolveUncertainOrder = jest.fn().mockResolvedValue(UncertainOrderResolution.UNAVAILABLE); + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue({ + supportedCommands: ['withdraw'], + executeOrder: jest.fn(), + checkCompletion: jest.fn(), + validateParams: jest.fn(), + resolveUncertainOrder, + }); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + + await service['resolveUncertainOrders'](); + + jest.advanceTimersByTime(30 * 60_000 - 1); + await service['resolveUncertainOrders'](); + expect(resolveUncertainOrder).toHaveBeenCalledTimes(1); + + jest.advanceTimersByTime(1); + await service['resolveUncertainOrders'](); + expect(resolveUncertainOrder).toHaveBeenCalledTimes(2); + }); + it('caps the cooldown interval at thirty minutes no matter how old the order is', async () => { // Pins the cap to the millisecond. An 8-hour-old order's uncapped wait would be 48 minutes at the // first pass and 51 by the time of the boundary check — either way far past the cap, so a lookup at // exactly 30 minutes can only come from it. Requiring no lookup a millisecond earlier leaves the cap // no other whole-millisecond value to take, and landing on the boundary pins `<` against `<=`. const resolveUncertainOrder = stubResolver(); - const order = uncertainOrder({ created: new Date(Date.now() - 8 * 60 * 60_000) }); + const order = uncertainOrder({ + created: new Date(Date.now() - 8 * 60 * 60_000), + // Long bound on purpose (see above): this pins the thirty-minute cap, not a deadline. + action: { id: 233, system: 'Kraken', command: 'withdraw' } as LiquidityManagementOrder['action'], + }); jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); await service['resolveUncertainOrders'](); @@ -651,6 +1196,29 @@ describe('LiquidityManagementPipelineService', () => { }); }); + describe('checkRunningOrders', () => { + it('quarantines a running order whose command is no longer registered', async () => { + // Without the null guard in checkOrder this would TypeError, land only in logger.error, and leave the + // order stuck in IN_PROGRESS with no automatic or manual exit. OrderOutcomeUnknownException is the + // same path startNewOrders already uses for unknown outcomes — quarantine, not a hang. + const order = Object.assign(new LiquidityManagementOrder(), { + id: 11, + status: LiquidityManagementOrderStatus.IN_PROGRESS, + correlationId: 'dfx-lm-11', + action: { id: 233, system: 'Scrypt', command: 'sell-if-deficit' }, + }); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + jest.spyOn(orderRepo, 'save').mockImplementation(async (o: LiquidityManagementOrder) => o); + jest.spyOn(actionIntegrationFactory, 'getIntegration').mockReturnValue(null); + + await service['checkRunningOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + expect(order.errorMessage).toMatch(/no registered integration.*Scrypt\/sell-if-deficit/s); + expect(notificationService.sendMail).toHaveBeenCalled(); + }); + }); + describe('resolveUncertainOrderManually', () => { const VERIFIED_DTO = { noExecutionVerified: true, verificationReference: 'venue console, ticket OPS-42' }; diff --git a/src/subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.ts b/src/subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.ts index a25617b6ba..8c74b457fe 100644 --- a/src/subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.ts +++ b/src/subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.ts @@ -7,7 +7,7 @@ import { Util } from 'src/shared/utils/util'; import { MailContext, MailType } from 'src/subdomains/supporting/notification/enums'; import { MailRequest } from 'src/subdomains/supporting/notification/interfaces'; import { NotificationService } from 'src/subdomains/supporting/notification/services/notification.service'; -import { In } from 'typeorm'; +import { In, IsNull } from 'typeorm'; import { ResolveUncertainOrderDto } from '../dto/resolve-uncertain-order.dto'; import { LiquidityManagementOrder } from '../entities/liquidity-management-order.entity'; import { LiquidityManagementPipeline } from '../entities/liquidity-management-pipeline.entity'; @@ -17,6 +17,7 @@ import { OrderNotNecessaryException } from '../exceptions/order-not-necessary.ex import { OrderNotProcessableException } from '../exceptions/order-not-processable.exception'; import { OrderOutcomeUnknownException } from '../exceptions/order-outcome-unknown.exception'; import { LiquidityActionIntegrationFactory } from '../factories/liquidity-action-integration.factory'; +import { LiquidityActionIntegration } from '../interfaces'; import { LiquidityManagementOrderRepository } from '../repositories/liquidity-management-order.repository'; import { LiquidityManagementPipelineRepository } from '../repositories/liquidity-management-pipeline.repository'; import { LiquidityManagementRuleRepository } from '../repositories/liquidity-management-rule.repository'; @@ -321,10 +322,28 @@ export class LiquidityManagementPipelineService { /** * Resolve orders quarantined as UNCERTAIN by asking the venue what actually happened. * - * This only ever observes — it must not re-send anything. An order leaves quarantine when the venue - * either confirms it knows the reference (back to IN_PROGRESS, the normal completion check takes over) or - * demonstrably does not (FAILED, so the rule may plan anew from a fresh balance). Anything inconclusive - * stays put: an order nobody can account for is safer parked than retried. + * This only ever observes or cancels — it must never re-send anything. An order leaves quarantine when the venue + * either confirms it knows the reference *and* a registered command still exists to check its completion + * (back to IN_PROGRESS, the normal completion check takes over), or demonstrably does not (FAILED, so the + * rule may plan anew from a fresh balance). A venue-side SENT for a command that is no longer registered + * deliberately stays quarantined: without a completion check, returning it to IN_PROGRESS would leave it + * with no exit, so the way out remains the existing automatic cancel/abandon path + * (`cancelOutstanding` / `unresolvableTooLong`) — not an operator. Anything inconclusive stays put, and past + * the abandon bound for its kind of request a cancellation is attempted — because a rule parked forever is + * the worse failure. For a cancellable request it is given up as FAILED only once the venue has confirmed + * that nothing under this order can still execute. A Scrypt withdrawal cannot be cancelled at all, so there + * the bar is lower by design: the venue answered and did not name the reference, without any completeness + * check on that answer — a repeated payout goes to a DFX-owned address, whereas insisting on completeness + * would strand the order. Age decides when it is worth trying to clean up; what the venue says decides + * whether giving up is safe. So the bound is not a deadline after which the order is certainly gone. For Scrypt + * (and any system whose adapter implements `resolveUncertainOrder`), an order whose references the venue + * will not yet settle keeps waiting past it — on the venue, not on an operator: reconciliation resolves by + * system, so a renamed or removed command still reaches the adapter and can be observed or cancelled there. + * Returning that order to the normal pipeline, however, still requires `getIntegration` (registered command). + * Systems whose adapter omits `resolveUncertainOrder` have no automatic venue path; without a pending + * release they are skipped every pass until an operator acts (`releasePending`). An operator can still + * release sooner as a shortcut; where the adapter can ask, the mechanism that ends the wait is the venue + * answering (or confirming absence) on a later pass. */ private async resolveUncertainOrders(): Promise { // First: anything this process observed and could not write. Retried before new lookups, because an @@ -348,20 +367,43 @@ export class LiquidityManagementPipelineService { // pending release bypasses the wait entirely: a manual release must complete on the next tick, and // its own venue-wait runs on its own clock. if (!releasePending) { + const lastAttemptEnd = this.uncertainResolveAttempts.get(order.id); + const abandonableAt = order.getAbandonableAt(); const ageMs = Date.now() - order.created.getTime(); + + // How long the last lookup had left before this order became abandonable. Negative once that lookup + // itself ran after the deadline, and Infinity when there is no deadline or no previous lookup to + // measure from. + const deadlineHeadroomMs = + abandonableAt && lastAttemptEnd ? abandonableAt.getTime() - lastAttemptEnd.getTime() : Infinity; + const intervalMs = Math.min( Math.max(ageMs / 10, UNCERTAIN_RESOLVE_MIN_INTERVAL_MS), UNCERTAIN_RESOLVE_MAX_INTERVAL_MS, + // The throttle may never outlast the deadline it has to keep. This same pass is what abandons an + // order whose bound has run out, so a wait reaching past that bound postpones the abandonment beyond + // the ceiling the bound exists to impose — a trade quarantined when it was already eight hours old + // would be given up after thirty minutes instead of five, with nothing here saying so. + // + // Taken as-is while there is headroom, deliberately without the floor: a lookup that ended shortly + // before the deadline has less than a floor's worth of time left, and imposing a full interval on it + // would schedule the next pass after the deadline — the very overshoot this cap prevents, caused by + // the cap. It costs at most one extra lookup, because the pass it permits is the one that gives the + // order up. + // + // Past the deadline the floor governs instead: there is no deadline left to protect, and a + // cancellation the venue will not confirm must not retry on every ten-second tick. + deadlineHeadroomMs > 0 ? deadlineHeadroomMs : UNCERTAIN_RESOLVE_MIN_INTERVAL_MS, ); - const lastAttemptEnd = this.uncertainResolveAttempts.get(order.id); if (lastAttemptEnd && Date.now() - lastAttemptEnd.getTime() < intervalMs) continue; } try { - // Null when the action's system or command is no longer registered at all — an order can outlive the - // adapter that made it, and dereferencing that would throw here on every pass, forever. - const actionIntegration = this.actionIntegrationFactory.getIntegration(order.action); + // Null only when the action's *system* has no adapter at all. Command registration is ignored here: + // a quarantined order can outlive a command rename/removal, and reconciliation still needs whoever + // can ask that venue. Execution of new orders keeps using getIntegration (registered commands only). + const actionIntegration = this.actionIntegrationFactory.getReconciliationIntegration(order.action); if (!actionIntegration?.resolveUncertainOrder) { // The one exception to "a release waits for the venue": there is no lookup for this order at all, @@ -369,6 +411,12 @@ export class LiquidityManagementPipelineService { // operator's judgement is all there is, which is why the assertion behind it is required. if (releasePending && (await this.completeNotSentRelease(order, 'no integration can look it up'))) anyChanged = true; + // Deliberately no automatic abandon here. Without an integration the venue is never asked, so the + // only thing left would be the clock — and the safety of abandoning rests on the rule replanning + // from a balance that reflects an execution which did happen. That holds for an exchange read live + // at plan time; it does not hold for a chain balance that omits unconfirmed transactions, nor for a + // bank balance carried over from the last imported batch. Abandoning on the clock alone would be + // guessing with the one class of order nothing here can observe. continue; } @@ -383,7 +431,30 @@ export class LiquidityManagementPipelineService { } if (resolution === UncertainOrderResolution.SENT) { - if (await this.applyConfirmedObservation(order)) { + // Reconciliation looked up by system alone so an unregistered command can still be *observed* — + // that is intentional and stays that way (see getReconciliationIntegration above). Putting the + // order back into IN_PROGRESS is a different decision: the normal completion path uses the strict + // getIntegration (registered commands only). An order whose command is gone would land in + // IN_PROGRESS with no adapter that can checkCompletion — every checkRunningOrders pass would + // TypeError, never quarantine, and the automatic abandon path would never run. Observing is + // allowed for anyone who can ask the venue; advancing out of quarantine is only allowed for + // whoever can also finish the job. Leave it UNCERTAIN so cancelOutstanding / unresolvableTooLong + // still apply once the bound is reached. + if (!this.actionIntegrationFactory.getIntegration(order.action)) { + this.logger.warn( + `Uncertain liquidity order ${order.id} stays quarantined: venue confirmed it was sent ` + + `(system ${order.action.system}, command ${order.action.command}), but no registered command ` + + `can check its completion — returning it to IN_PROGRESS would leave it with no exit`, + ); + + // A SENT for a command that no longer exists is a real observation ("the reference exists") but + // not one that can return to normal operation. The only remaining exit is the same as for an + // unresolvable case: after the bound elapses, cancel and abandon. resolveUncertainOrder answers + // "does the reference exist", not "is it still open" — so SENT stays SENT permanently, and the + // cleanup path must not depend on a future status change; it has to run independently of whether + // the command is still registered. + if (await this.attemptQuarantineCleanup(order, actionIntegration)) anyChanged = true; + } else if (await this.applyConfirmedObservation(order)) { anyChanged = true; this.logger.info(`Uncertain liquidity order ${order.id} resolved: venue confirmed it was sent`); } @@ -396,14 +467,23 @@ export class LiquidityManagementPipelineService { // checked independently and released the order on that basis. Two negatives, one of them from a // person who looked: that is what this release was waiting for. if (await this.completeNotSentRelease(order, 'the venue has no record of it either')) anyChanged = true; + } else if (releasePending && !order.correlationId) { + // Nothing ever went out under a reference, so no lookup can produce an answer — the same dead end + // as an order with no integration, and the same reason not to make somebody who already checked + // wait out a clock. Without this the release would sit through the full unreachable-venue wait, + // because an unaskable order now reports UNAVAILABLE rather than an answer. + if (await this.completeNotSentRelease(order, 'no reference exists to look it up')) anyChanged = true; } else if (releasePending && order.releaseWaitedOutVenue()) { // Nobody has been able to ask this venue anything for long enough. Waiting more does not make an // answer likelier; it only keeps an order a person has verified by hand out of reach. if (await this.completeNotSentRelease(order, 'the venue could not be reached for long enough')) anyChanged = true; + } else if (await this.attemptQuarantineCleanup(order, actionIntegration)) { + // see attemptQuarantineCleanup + anyChanged = true; } - // Otherwise — a venue that cannot be asked yet, or an inconclusive answer with nobody having - // released the order — nothing changes and it keeps blocking. + // Otherwise — an order still inside the window in which its request could be live — nothing changes + // and it keeps blocking. } catch (e) { // a failing lookup must never promote the order out of quarantine this.logger.error(`Error in resolving uncertain liquidity order ${order.id}:`, e); @@ -413,13 +493,54 @@ export class LiquidityManagementPipelineService { return anyChanged; } + /** + * Attempt cancel-and-abandon for a quarantined order that has outlived its bound. + * + * Called from two places that share the same exit and must not invent two clocks for it: the ordinary + * end of the if/else-if chain (inconclusive lookup, bound reached), and the SENT branch when the venue + * confirmed the reference but no registered command remains to finish the job. The deadline check lives + * here — not at either call site — so both paths apply the same bound, and so the SENT path can invoke + * cleanup without re-entering the chain or falling through into completeNotSentRelease. + * + * Returns whether the order was abandoned (and thus whether the caller should count a state change). + */ + private async attemptQuarantineCleanup( + order: LiquidityManagementOrder, + actionIntegration: LiquidityActionIntegration, + ): Promise { + if (!order.unresolvableTooLong()) return false; + + // Old enough that cleaning it up is worth attempting, and nobody has released it. Leaving it here + // forever is not the careful option — the rule then never runs again and the venue stops being + // served entirely. Trade and withdraw both reach this path: the integration decides what settles the + // question (cancel every reference, or — for a withdrawal, which cannot be cancelled — a venue reply + // that does not name the reference) and returns the reason wording the abandon step will record. + // + // What stands in the way of giving up is never the order itself but the possibility of a request + // still executing: hand the funds back and a late fill spends them twice. So rather than + // estimating when that can no longer happen — these are orders nothing expires, so age proves + // nothing — the possibility is removed. Cancelling is the opposite of re-sending and cannot create + // anything, and once the venue confirms nothing can execute, abandoning is a fact rather than a + // guess. Refuses to settle, or cannot be reached? Then nothing changes and the order waits on the + // venue (an operator is only a shortcut past the next automatic pass). + const because = await actionIntegration.cancelOutstanding?.(order); + if (!because) return false; + + return this.abandonUncertainOrder(order, because); + } + /** * Put a not-sent conclusion into effect: the order becomes an ordinary failure and its rule may plan anew. * - * The only place an order leaves quarantine downwards. Everything that reaches here has either a venue + * The evidence-based way out of quarantine downwards. Everything that reaches here has either a venue * verdict behind it, or a person who checked plus a venue that has no record — never a single judgement on * its own. The two exceptions are about liveness, not evidence: a venue nothing can ask, and one that has * answered nothing for long enough. Silence there stops vetoing the person who checked; it proves nothing. + * + * The other way out is {@link abandonUncertainOrder}, which concludes nothing about the send itself and + * rests instead on the venue confirming that nothing can still execute — so that an order nobody releases + * is not held by that alone. It is not a guarantee against blocking: where the venue will not confirm, or + * cannot be asked to cancel, this release stays the only way out. */ private async completeNotSentRelease(order: LiquidityManagementOrder, because: string): Promise { // The release this pass looked at, captured before the entity is mutated. Ending an order is the one @@ -436,6 +557,44 @@ export class LiquidityManagementPipelineService { return true; } + /** + * Abandon an order with nothing left outstanding at the venue, so its rule runs again. + * + * The way out of quarantine that rests on no conclusion about whether the request was ever sent — that + * stays unknown, which is why `because` may only ever describe what the venue confirmed, never that + * nothing was sent. The row must not claim an observation nobody made. + * + * The clock does not release anything on its own: it only decides when cleaning up is worth attempting. + * What permits the release is the venue confirming that none of the order's references can execute. + * + * Reached only after the venue has confirmed that none of the order's references can execute any more, so + * what it ends is a wait, not an open question. + * + * Logged as a warning, not an info. Nothing here is routine — an order reaching this point means the venue + * lost track of a request past its bound — and the entry is what makes that visible without an operator + * having to be the mechanism that unblocks it. + */ + private async abandonUncertainOrder(order: LiquidityManagementOrder, because: string): Promise { + // Usually null, because a pending release is handled by an earlier branch for every answer that concerns + // it — but not always: this branch has no release condition of its own, so an order released while the + // venue was unreachable reaches it with the marker still set, and is then given up on the cancellation + // rather than on the release. Whichever it is, the value read here is the one narrowed on, which is the + // point: an operator may write a release between that read and this write, and that release carries an + // audited reason and is owed one more venue answer. Without the narrowing this write — which rests on the + // venue's cancellation but on no evidence about whether the request was ever sent — would silently + // overwrite the one resting on a person. (The reason itself survives either way: the abandonment prefixes + // the existing message rather than replacing it.) + const examined = order.notSentRecheckDue ?? null; + + order.abandonUncertain(`${order.errorMessage} (abandoned ${new Date().toISOString()}: ${because})`); + + if (!(await this.leaveQuarantine(order, examined))) return false; + + this.logger.warn(`Uncertain liquidity order ${order.id} abandoned: ${because}`); + + return true; + } + /** * Record that the venue holds this order — and make sure that fact lands somewhere durable. * @@ -576,8 +735,16 @@ export class LiquidityManagementPipelineService { { id: order.id, status: LiquidityManagementOrderStatus.UNCERTAIN, - // narrowed by the caller when the outcome depends on WHICH pending release was examined - ...(expectedRecheckDue !== undefined ? { notSentRecheckDue: expectedRecheckDue } : {}), + // Narrowed by the caller when the outcome depends on WHICH pending release was examined. + // + // `IsNull()` rather than a bare null: TypeORM renders a raw null in a where object as `= NULL` + // (invalidWhereValuesBehavior.null defaults to "ignore", which falls through to an equality), and + // `x = NULL` is UNKNOWN in SQL, so it matches nothing — not even the row whose column really is + // NULL. "No release was pending" is the ordinary case for every caller here, so without this the + // narrowed update would silently never affect a row and report itself as a lost race. + ...(expectedRecheckDue !== undefined + ? { notSentRecheckDue: expectedRecheckDue === null ? IsNull() : expectedRecheckDue } + : {}), }, { status: order.status, @@ -591,7 +758,21 @@ export class LiquidityManagementPipelineService { ); if (!result.affected) { - this.logger.info(`Uncertain liquidity order ${order.id} was already resolved elsewhere, skipping`); + // Two very different situations, and this write cannot tell them apart: either somebody resolved the + // order between the read and here — a race, which the next pass simply sees — or the narrowing above + // matched no row and never will, in which case this repeats on every pass while the order stays exactly + // as it is. Claiming the benign one was wrong: a release timestamp written with microsecond precision + // cannot be matched by a JavaScript Date, which carries milliseconds, and one written by hand in SQL held + // a live order for hours behind the reassuring version of this line. + // + // So it names both and says what to look for. A warning rather than info because a race is rare and + // self-correcting while the other case is a silent permanent block, and this line is the only trace it + // leaves — the whole failure this branch exists to end. + this.logger.warn( + `Uncertain liquidity order ${order.id} was not updated: either it was resolved elsewhere, or the ` + + `quarantine narrowing matched no row. Repeating on every pass means the latter — the order is stuck.`, + ); + return false; } @@ -628,9 +809,12 @@ export class LiquidityManagementPipelineService { * Release a quarantined order by hand, after somebody checked the venue directly. * * Reconciliation can only ever confirm that a reference exists; it never concludes the opposite, because - * no venue reply proves "this was never accepted". Without this path a genuinely unsent request would - * block its rule forever. Guarded like the payout subdomain's retry: the caller must assert the check and - * name where it happened, and the assertion is recorded on the order. + * no venue reply proves "this was never accepted". An order that can at least be cancelled is given up + * once the venue confirms nothing can still execute, so this path is what keeps the rest moving: orders no + * integration can look up, and venues that will not settle them, which nothing here would otherwise + * release. Guarded like the + * payout subdomain's retry: the caller must assert the check and name where it happened, and the + * assertion is recorded on the order. */ async resolveUncertainOrderManually( orderId: number, @@ -737,7 +921,20 @@ export class LiquidityManagementPipelineService { } private async checkOrder(order: LiquidityManagementOrder): Promise { + // A running order whose command is no longer registered cannot be completed through the normal path. + // getIntegration returns null for unregistered commands; without this guard the next line would throw a + // TypeError that checkRunningOrders only logs — the order would stay IN_PROGRESS forever, outside + // quarantine, where neither automatic abandon nor the manual release endpoint can reach it. Quarantining + // via OrderOutcomeUnknownException is not a Scrypt special case: for any system, a live order without an + // adapter is better in UNCERTAIN (automatic and manual exits both apply) than in a state where nothing + // acts on it. const actionIntegration = this.actionIntegrationFactory.getIntegration(order.action); + if (!actionIntegration) { + throw new OrderOutcomeUnknownException( + `Liquidity order ${order.id} has no registered integration for ${order.action.system}/${order.action.command} that can check its completion`, + ); + } + const isComplete = await actionIntegration.checkCompletion(order); if (isComplete) { From b067f1d08d2d4f20979363e93feede5ff78469e1 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Sat, 1 Aug 2026 11:18:50 +0200 Subject: [PATCH 3/4] Log the rejected value and the caller on a rejected request (#4562) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Name the caller in the log line of a rejected request A rejected request is logged with its reason but with nothing about who sent it. On an endpoint that runs without authentication there is then no way to tell a partner integration, one of our own apps and a third-party script apart, which leaves a recurring rejection without an owner who could fix it. The WARN line now carries the X-Client value, the requesting site (Origin, or the origin of Referer - never its path or query, which can hold personal data or tokens) and the user agent. All three are client-supplied and unauthenticated, so they are a diagnostic hint, never an identity, and nothing is gated on them. Each part is masked to the same standard as the rest of the logs, capped per header, and stripped of anything that could break the line, so a crafted header cannot forge a log entry of its own. * Log the values a rejected request sent A validation message names the field and the values it accepts, not the value that arrived. A client sending one wrong constant is therefore visible as a steady rejection count and stays unidentifiable: the same line is produced whether the field was missing, misspelled or set to a value from a neighbouring enum. The global ValidationPipe now raises a ValidationFailedException that carries the raw ValidationError[] alongside the response, and the exception filter renders the rejected values into its WARN line. The response is unchanged - the exception is built by the base factory and only re-raised, which a test pins field by field against the stock pipe. The rendering treats every value as untrusted input: redacted by field name, masked by value pattern, bounded in count, depth and length, and reduced to a summary for structured or oversized values, so a request body cannot be turned into a log dump through a validation failure. * Render a rejected value only where the field declares what it accepts The first pass rendered any scalar that failed validation and relied on a field-name denylist to hold back the sensitive ones. Review found three fields it does not hold back: `apiKey` and `masterKey` are not matched by it, `accountNumber` is not either because the list anchors `number` exactly, and a rejected `webhookUrl` or `redirectUri` would have carried its query string - the place a webhook credential lives - into the log verbatim. Widening the list would have been the wrong repair. It has to grow with every DTO that ever carries a credential, and the field it has not heard of yet is the one that leaks. The value is now rendered only where the constraint that failed declares a closed set of literals (`isEnum`, `isIn`). That is the case the log line cannot be read without, because one client sending one wrong constant looks exactly like another sending a different one, and it is bounded by what the field declares rather than by what happened to arrive. Every other field keeps its shape and loses its content: ``, ``, ``. Missing, null and empty stay visible for every field - there is nothing to disclose, and it is what separates a client that never sent the field from one that sent it wrong. The name-based redaction stays as a second layer over the fields it does cover. Also moves the oversize rule into `maskLogValue`, so a caller cannot pay for masking a string it is about to cut away; types the exception body as `Record` rather than `any`; and points the neighbouring payment-info spec at the pipe `main.ts` now installs, whose comment this change had made inaccurate. * Pin the two files this change completes in the coverage ratchet The gate's advisory step named them on this branch and not on the base: the new DTO spec is what carries `get-buy-quote.dto.ts` and `xor.validator.ts` to 100% on all four metrics. Unpinned they would stay unguarded, so a later change could erode the coverage this pull request created without turning the gate red. * Say what the literal-domain rule bounds, and what it does not The comment read as if declaring a closed set of values also bounded the value that arrives there. It does not: the declaration decides which fields may show their content at all, and what is then shown is still untrusted input, held only by the masking and the length cap. * Say what holds a rendered value, and cover the case that showed it did not The comment claimed that what gets rendered "stays masked and capped". That is true of a string, and only of a string: a number or a boolean is written out as it is - which is harmless, because being one is already the bound, but the sentence did not say so. The gap was possible because no test covered a non-string under a closed set of values, although the repository has such a field: `noExecutionVerified` is `@IsBoolean() @IsIn([true])`. Adds one. * Note the field name in the rule the previous commit already applied to it That commit put the field name through the same rendering as the value - a precaution, since no DTO today validates through a client-keyed object - but said only what it did to the value. Names it, and closes the paragraph it had left mid-line. * Reduce the Origin header to its origin as well Only the referring URL was cut down to its origin. `Origin` was passed through as it arrived - on the reasoning that it carries nothing else - and that is the reasoning this whole line is built to distrust: the header comes from the client, and a value that is not what it is supposed to be is exactly the one that must not reach the log with a query string on it. A crafted `Origin: https://example.com/x?token=…` went into the WARN line as sent, which is what the comment above it says cannot happen. Both headers now go through the same reduction, and an unparsable value is dropped either way. Two comments were left behind by the previous commit and are corrected with it: the summary over `describeRejectedValues` and the one in the exception filter both still claimed that everything rendered is masked and capped, which holds for a string and not for a number or a boolean. And the field name goes through `maskLogValue`, not through the whole of `renderValue` - "the same rendering as the value" overstated it. * Pick the first header that yields an origin, not the first one that is there Reducing `Origin` to its origin cost two things that were not meant to go: `Origin: null` - what a browser sends for an opaque origin, a sandboxed frame or a redirect across sites - is not a URL, so it fell into the catch and vanished, although having no origin to name is itself worth the line. And an `Origin` the client filled with something unparsable now suppressed the `Referer` too, because the choice was made on which header was present rather than on which one produced something: a caller that could have been named ended up as no caller at all. The headers are now tried in order and the first one that yields an origin wins, with the opaque literal kept as it is. Both cases are pinned by a test. Also drops a completeness claim from the neighbouring DTO spec: it demonstrates two rejections, it does not enumerate the ones that DTO can produce. * Pin that an opaque origin beats a referer, like any other origin does The rule is that the first header to yield an origin wins, and `null` is one - so a request that carries both an opaque `Origin` and a usable `Referer` is logged as opaque. That is intended and now written down in the comment, but nothing held it: the test covered the opaque header alone. * Make the origin cap test reach the cap The oversized origin it sent was not a URL, so `callerOrigin` dropped it before the cap could apply: the assertion that nothing floods the line held for a reason the test did not intend, and the origin cap itself was never exercised. It now sends a parsable URL that is too long and checks where it gets cut. * Cut a capped value between characters, not between code units `slice` counts UTF-16 units, so a value whose character straddles the cap lost half of a surrogate pair: the stray half went into the log right before the ellipsis and reaches a UTF-8 transport as a replacement character. Every caller inherits it - both caller headers and every rendered field value - since none of them filter what a client can send. The cut now walks characters. Pinned by a test that puts an emoji on the boundary. * Name the unit of a reported length, and the factory's return type `maskLogValue` reports an oversized value as ``, but N is `String.length` - UTF-16 code units, not characters. 257 astral characters are reported as 514. That is the very distinction the previous commit drew for the cut, so the label is renamed rather than the count converted: the number is the one `MAX_STRING` is compared against, and counting characters would mean walking the oversized string this branch exists to avoid. The same inaccuracy exists in `redact` and `format` (`<... N chars ...>`, `...(N chars)`), which are untouched here: those are shipped log formats and outside what this branch changes. `DetailedValidationPipe.createExceptionFactory` gets the explicit return type CONTRIBUTING.md asks for, matching the base signature. * Keep the whole warning on one line, and mask an IBAN by value Two gaps the caller markers and the rejected values were already closed against, but the line they join was not: The reason is masked, but not collapsed. Ninety-nine of the exceptions that produce it interpolate a value into their message, and some of those values come from the request (`Invalid address for ${field}: ${address}`), so a line break in one of them ends the log line and starts a second one that looks like a log entry of its own. It now goes through the same collapse and the same character-safe cut as everything else on the line - `slice` would have halved a surrogate pair at the cap the same way it did before the previous commit. Neither the name-based nor the value-based redaction covered an IBAN that is not under a field named for one. The rejected value of a field with a closed set of literals is rendered as it arrived, so a client putting an account number into such a field put it in the log; in the request trace the same value under any key the name list does not know reached it too. `maskValue` masks it by shape now, in one run or in the groups of four it is printed in, bounded so a transaction hash is still left intact. `singleLine` and `capCharacters` come out of `maskLogValue` for this, which leaves one definition of "cannot break a log line" in the file. `format` gets it as well: `JSON.stringify` escapes the control characters but leaves U+2028 and U+2029, which is the one way the request trace could still be split. * Drop the IBAN pattern, and bound the work the cap and the scan do The IBAN pattern is withdrawn. Matching an account number by shape needs the country to know where it ends: without it the grouped form either stops short of a 32-character IBAN and leaves the last group readable, or runs on and swallows the words behind it - both were measured, and the way out of both is a country-length table inside a log formatter. A value-shape list is also open by construction: an account number is one shape, a document number and a card are others, and the one that is not in the list yet is the one that leaks. What is left is the rule the branch already had - the value is only rendered where the field declares a closed set of literals, and it is masked by field name there. `capCharacters` walks to the cap instead of materializing the value first. It is reached with an exception message now, and one of those can be as large as the body it interpolated a value from; spreading that to render 500 characters cost a multiple of the message in heap. The reason is cut to a scan length before it is masked for the same reason, wide enough past the cap that a pattern starting inside it is still seen whole. `format` cuts on a character boundary too - it was the one section left that could halve a surrogate pair - and reports the section length in the unit it counts, as the per-string cap already does. Two comments claimed more than the code does: not every string on a trace line goes through the collapse, only the free-form values do. * Close the three ways the log line still let something through The client header was the one value on a trace line that was interpolated as it arrived. A header can carry U+0085, which `LINE_BREAKING` covers and no other step did, so it is rendered like every other value from the caller now, capped to a name rather than a payload. The reason's scan cut could expose what it split. Masking only shortens, so the part of a message that survives to the log can come from well past the visible cap - and a pattern that straddled the cut arrives halved, is no longer recognized, and its head is then among the characters that survive. A pattern length is read past what is kept and dropped again, which leaves only patterns that ended before the cut, and those were seen whole. The length comes from the patterns themselves, next to where they are declared. Reading the message can throw: the response body is whatever the thrower put there, and an array element that cannot be turned into a string takes `join` with it - before the response is sent. The line loses its reason instead. `format` cuts to its own budget again. That budget is in code units and `capCharacters` counts characters, so an astral section came out at twice it; the cut moves off a surrogate pair rather than counting past it. * Mask before collapsing, and stop trying to bound the scan Masking ran after the collapse, so a control character sitting inside a pattern took the pattern with it: the collapse turned it into a space, the pattern no longer matched, and an address was logged in full. It was measured on both the reason and the client header. Masking runs first now, on the value as it arrived. The scan margin is withdrawn. It cut the message before masking so the work was bounded, and every version of it cut somewhere a pattern could be open - a pattern crossing the cut arrives halved, is no longer recognized, and masking what precedes it shortens the text enough to pull its head into view. Two attempts at a margin produced two ways for that to happen. The message is masked whole again, as it was before this branch; what is left is the cap, which is what this branch actually needed to bound. The request target was the last free-form value that reached a line as it arrived. `maskUrl` collapses it too, which is one line where the query is already being dropped, and makes the sentence about this file true. The response now goes out before the line is written. Everything the line renders comes from the request or from the thrower, and reading either can throw - which left the caller with no response rather than with a line missing a detail. An unreadable exception body gets the generic one instead of none. Two comments claimed more than they had to: a message *can* interpolate a request value, and what the field-name masking covers is stated without a claim about every DTO in the repository. * Let the field say its rejected value may be logged Rendering the value wherever the constraint declares a closed set of literals was the wrong rule, and it is the one thing this branch does that `develop` does not: `develop` logs no rejected values at all. A constraint bounds what is accepted, not what a client sends, and a rejected value is by definition outside it - so `paymentMethod` rendered an account number, a card number, a passport number and an API key just as readily as it rendered a wrong constant. Every attempt to catch those by shape has failed in a new way, because that list is open. `@LogRejectedValue()` closes it from the other side. The field declares that its wrong values are constants of the program, and only a field that says so has its value rendered; everything else keeps its shape and loses its content, including every field that exists today. The two payment-method fields and the personal IBAN provider carry the marker, which is what the endpoint this started from needs. An error without the object it came from renders nothing, so a value cannot arrive through a `ValidationError` that never passed a DTO. `maskLogValue` masks a value whole when removing the control characters is what reveals a pattern in it. A control character splits the pattern so it no longer matches, and the collapse then puts the halves next to each other in the line; a single value is not a sentence, so masking more of it than the pattern costs nothing. Two comments still said more than they had to: the response is already sent by the time the reason is read, and what is masked by value are the three patterns declared above, not personal data in general. * Render a declared constant, never the string that arrived The marker said a field's value may be shown; it did not say which values, so a client could still put an account number, a card number or an API key into a field that carries it and have it logged in full. The marker now carries the set those values may come from, and only a value in that set is rendered - matched without regard to case, and written out of the set rather than out of the request, so what reaches the line is a constant of this program either way. For a field taking the fiat payment methods that set is the full payment-method union: its crypto member is exactly what a client sends there by mistake, which is the case these lines exist for. The personal IBAN provider loses the marker again - there is no wider set its wrong values come from, so it would never render anything. With that, nothing client-composed reaches the line and the masking is no longer what protects it, which is what every attempt so far had rested on. `singleLine` removes what could break a line instead of replacing it, and runs before the masking everywhere: a character placed inside a pattern used to leave the pattern split around whatever replaced it, so the masking no longer saw it - in the reason, in the request target and in the trace bodies, not only where the per-value guard reached. Removing it puts the pattern back together first. That guard is gone with it. * Give the provider its values back, and keep the reading honest The personal IBAN provider was dropped from the declarations on the grounds that no wider set exists for it. That was wrong: the matching ignores case, so the field's own values are exactly what makes `frick` readable as `Frick` - the mistake this line is for. It declares them again. A numeric enum object carries its reverse mapping, so `Object.values` on one returns the member names alongside the values and a request sending a name would have matched. Only the values are taken now. A declared constant is written by this code rather than by the request, but it goes through the same rendering as everything else on the line - there is no reason for the one value on it that is not checked to be the one a hand wrote. The status is resolved before anything else is read and falls back to a server error when the exception cannot answer or answers with something Express will not send; the request is read after the response has gone out, since the response never needed it. Two comments described the replaced behaviour rather than the removal that took its place. * Let a declared value past the field name, and mask on both sides of the removal The provider declaration was inert. `personalIbanProvider` carries `iban` in its name, so the name-based redaction answered first and the value never reached the declaration - the fix landed and changed nothing. A declared match now comes first: what it renders is a constant of this program, and the field's name says nothing about a value that was never the client's. Removing what breaks a line breaks a pattern in the other direction. It joins what stood on either side, so an address followed by a control character and one more digit comes out as one run, where it no longer ends on a word boundary and the masking no longer sees it - the mirror image of the case the removal was introduced for. Both passes run now, around the removal rather than on one side of it, in the one place that renders free-form text for a line. A second pass cannot invent a match: what the first leaves behind carries no alphanumerics. The response body is the generic one whenever the status being sent is not the one the exception names, so a replaced status no longer ships a body that contradicts it, and a message that cannot be read falls back to the status rather than out of the method. The line is written inside its own guard. The response is already out by then, so a failure to describe it - the logger included, which is the one thing that could not report it - ends there instead of travelling back to the caller. 1xx leaves the accepted range: it is not a final response, so it is not one this can send. --- jest.coverage-gate.config.js | 3 + src/main.ts | 7 +- .../log-rejected-value.decorator.spec.ts | 77 +++++ .../log-rejected-value.decorator.ts | 71 +++++ .../__tests__/exception.filter.spec.ts | 179 +++++++++++ src/shared/filters/exception.filter.ts | 111 +++++-- .../__tests__/api-trace.middleware.spec.ts | 122 +++++++- .../middlewares/api-trace.middleware.ts | 97 +++++- .../detailed-validation.pipe.spec.ts | 277 ++++++++++++++++++ src/shared/pipes/detailed-validation.pipe.ts | 135 +++++++++ .../utils/__tests__/request-caller.spec.ts | 92 ++++++ src/shared/utils/request-caller.ts | 70 +++++ .../get-buy-payment-info.dto.spec.ts | 40 ++- .../dto/__tests__/get-buy-quote.dto.spec.ts | 51 ++++ .../buy/dto/get-buy-payment-info.dto.ts | 7 +- .../routes/buy/dto/get-buy-quote.dto.ts | 5 +- 16 files changed, 1304 insertions(+), 40 deletions(-) create mode 100644 src/shared/decorators/__tests__/log-rejected-value.decorator.spec.ts create mode 100644 src/shared/decorators/log-rejected-value.decorator.ts create mode 100644 src/shared/pipes/__tests__/detailed-validation.pipe.spec.ts create mode 100644 src/shared/pipes/detailed-validation.pipe.ts create mode 100644 src/shared/utils/__tests__/request-caller.spec.ts create mode 100644 src/shared/utils/request-caller.ts create mode 100644 src/subdomains/core/buy-crypto/routes/buy/dto/__tests__/get-buy-quote.dto.spec.ts diff --git a/jest.coverage-gate.config.js b/jest.coverage-gate.config.js index 3374a71660..093831ff80 100644 --- a/jest.coverage-gate.config.js +++ b/jest.coverage-gate.config.js @@ -62,12 +62,14 @@ const PINNED_LOGIC = [ 'src/shared/auth/allow-tfa-pending.decorator.ts', 'src/shared/auth/get-jwt.decorator.ts', 'src/shared/auth/user-role.enum.ts', + 'src/shared/decorators/log-rejected-value.decorator.ts', 'src/shared/services/typeorm-logger.ts', 'src/shared/utils/bitbox-ascii.util.ts', 'src/shared/utils/cron.ts', 'src/shared/utils/custom-cron-expression.ts', 'src/shared/utils/request-client.ts', 'src/shared/validators/is-ssrf-safe-url.validator.ts', + 'src/shared/validators/xor.validator.ts', 'src/subdomains/core/accounting/controllers/ledger.controller.ts', 'src/subdomains/core/accounting/dto/ledger-account.dto.ts', 'src/subdomains/core/accounting/dto/ledger-dto.mapper.ts', @@ -92,6 +94,7 @@ const PINNED_LOGIC = [ 'src/subdomains/core/aml/enums/scorechain-outcome.enum.ts', 'src/subdomains/core/aml/services/transaction-aml-check.service.ts', 'src/subdomains/core/buy-crypto/process/exceptions/abort-batch-creation.exception.ts', + 'src/subdomains/core/buy-crypto/routes/buy/dto/get-buy-quote.dto.ts', 'src/subdomains/core/buy-crypto/routes/buy/dto/personal-iban-provider.enum.ts', 'src/subdomains/core/custody/dto/output/custody-order-history.dto.ts', 'src/subdomains/core/custody/enums/custody.ts', diff --git a/src/main.ts b/src/main.ts index 48e445c257..96976213a5 100644 --- a/src/main.ts +++ b/src/main.ts @@ -4,7 +4,7 @@ // internal HTTP usage is auto-instrumented. import './tracing'; import './polyfills'; // registers global EventSource for @arkade-os/sdk; see src/polyfills.ts -import { ValidationPipe, VersioningType } from '@nestjs/common'; +import { VersioningType } from '@nestjs/common'; import { NestFactory } from '@nestjs/core'; import { WsAdapter } from '@nestjs/platform-ws'; import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; @@ -20,6 +20,7 @@ import { AppModule } from './app.module'; import { Config, Environment } from './config/config'; import { ApiExceptionFilter } from './shared/filters/exception.filter'; import { apiTraceMiddleware, maskUrl } from './shared/middlewares/api-trace.middleware'; +import { DetailedValidationPipe } from './shared/pipes/detailed-validation.pipe'; import { DfxLogger } from './shared/services/dfx-logger'; import { AccountChangedWebhookDto } from './subdomains/generic/user/services/webhook/dto/account-changed-webhook.dto'; import { @@ -86,7 +87,9 @@ async function bootstrap() { defaultVersion: [Config.defaultVersion], }); app.useGlobalPipes( - new ValidationPipe({ + // Same validation and same 400 body as the stock ValidationPipe; it additionally carries the + // rejected values through to the exception filter's log line. + new DetailedValidationPipe({ whitelist: true, transformOptions: { exposeUnsetFields: false, diff --git a/src/shared/decorators/__tests__/log-rejected-value.decorator.spec.ts b/src/shared/decorators/__tests__/log-rejected-value.decorator.spec.ts new file mode 100644 index 0000000000..3bc26c70fc --- /dev/null +++ b/src/shared/decorators/__tests__/log-rejected-value.decorator.spec.ts @@ -0,0 +1,77 @@ +import { LogRejectedValue, loggableRejectedValues } from 'src/shared/decorators/log-rejected-value.decorator'; + +enum Mode { + FAST = 'Fast', + SLOW = 'Slow', +} + +enum Speed { + LOW = 0, + HIGH = 1, +} + +class Declaring { + @LogRejectedValue(Mode) + mode: string; + + @LogRejectedValue(['Bank', 'Crypto']) + method: string; + + free: string; +} + +class Silent { + mode: string; +} + +class Numeric { + @LogRejectedValue(Speed) + speed: number; +} + +class Extending extends Declaring { + @LogRejectedValue([1, 2, true]) + own: string; +} + +describe('LogRejectedValue', () => { + it('takes the values of an enum object', () => { + expect([...(loggableRejectedValues(Declaring, 'mode') ?? [])]).toEqual([ + ['fast', 'Fast'], + ['slow', 'Slow'], + ]); + }); + + it('takes a plain list, including numbers and booleans', () => { + expect([...(loggableRejectedValues(Extending, 'own') ?? [])]).toEqual([ + ['1', '1'], + ['2', '2'], + ['true', 'true'], + ]); + }); + + it('leaves the reverse mapping of a numeric enum out', () => { + // `{ LOW: 0, HIGH: 1 }` reads back as `['LOW', 'HIGH', 0, 1]`: the member names are not values + // the field accepts, and taking them would match a request that sent one of them. + expect([...(loggableRejectedValues(Numeric, 'speed') ?? [])]).toEqual([ + ['0', '0'], + ['1', '1'], + ]); + }); + + it('reports nothing for a property that declared nothing', () => { + expect(loggableRejectedValues(Declaring, 'free')).toBeUndefined(); + expect(loggableRejectedValues(Silent, 'mode')).toBeUndefined(); + }); + + it('reports nothing for anything that is not a class', () => { + expect(loggableRejectedValues(undefined, 'mode')).toBeUndefined(); + expect(loggableRejectedValues({ mode: 1 }, 'mode')).toBeUndefined(); + expect(loggableRejectedValues('Declaring', 'mode')).toBeUndefined(); + }); + + it('inherits what a parent declared without a subclass reaching back into it', () => { + expect(loggableRejectedValues(Extending, 'mode')?.get('fast')).toBe('Fast'); + expect(loggableRejectedValues(Declaring, 'own')).toBeUndefined(); + }); +}); diff --git a/src/shared/decorators/log-rejected-value.decorator.ts b/src/shared/decorators/log-rejected-value.decorator.ts new file mode 100644 index 0000000000..f78dac7a53 --- /dev/null +++ b/src/shared/decorators/log-rejected-value.decorator.ts @@ -0,0 +1,71 @@ +// Per class, the values each property may contribute to a log line, keyed by property and held on +// the class itself. Shared between the decorator and its reader so the two can never drift. +const LOG_REJECTED_VALUE = Symbol('logRejectedValue'); + +type Loggable = { [LOG_REJECTED_VALUE]?: Map> }; + +/** The values a field may render, as an enum object or a plain list. */ +export type LoggableValues = Record | readonly (string | number | boolean)[]; + +/** + * Declares which values of a DTO property may be written to a log line when the property is + * rejected. + * + * A rejection names the field and the values it accepts, never the one that came - so a client + * sending a wrong constant produces the same line as one sending nothing, and the fix, usually one + * constant in one client, cannot be named from the logs. Naming it needs the value; logging the + * value as it arrived does not work, because a constraint bounds what is accepted and a rejected + * value is by definition outside it. A field constrained to three payment methods rejects an + * account number exactly as readily as it rejects a typo. + * + * So the field declares a set instead, and only a value in that set is rendered - matched without + * regard to case, and rendered from the set rather than from the request, so what reaches the log + * is a constant of this program either way. Everything else keeps its shape and loses its content. + * + * The set to pass is the one a wrong value plausibly comes from and the field does not accept: for + * a field taking the fiat payment methods, the full payment-method union, whose crypto members are + * exactly what a client sends there by mistake. + */ +export function LogRejectedValue(values: LoggableValues): PropertyDecorator { + const loggable = new Map(declared(values).map(canonical)); + + return (target: object, property: string | symbol) => { + const type = target.constructor as Loggable; + + // A subclass starts from what it inherits and grows its own map, so declaring a property on it + // never reaches back into the class it extends. + const declared = Object.prototype.hasOwnProperty.call(type, LOG_REJECTED_VALUE) + ? (type[LOG_REJECTED_VALUE] as Map>) + : new Map(type[LOG_REJECTED_VALUE] ?? []); + + declared.set(property, loggable); + Object.defineProperty(type, LOG_REJECTED_VALUE, { value: declared, configurable: true }); + }; +} + +/** + * What the given property of the given class may render, or undefined if it declared nothing. The + * keys are lower-cased; the values are the constants as they are written in the code. + */ +export function loggableRejectedValues( + type: unknown, + property: string | symbol, +): ReadonlyMap | undefined { + if (typeof type !== 'function') return undefined; + + return (type as Loggable)[LOG_REJECTED_VALUE]?.get(property); +} + +// A numeric enum object carries its reverse mapping as well, so its own member names appear among +// its values - `{ FAST: 0, SLOW: 1 }` reads back as `['FAST', 'SLOW', 0, 1]`. A name that maps to a +// number is one of those and is not a value the field ever accepts. +function declared(values: LoggableValues): (string | number | boolean)[] { + if (Array.isArray(values)) return [...values]; + + const members = values as Record; + return Object.values(members).filter((value) => typeof members[`${value}`] !== 'number'); +} + +function canonical(value: string | number | boolean): [string, string] { + return [`${value}`.toLowerCase(), `${value}`]; +} diff --git a/src/shared/filters/__tests__/exception.filter.spec.ts b/src/shared/filters/__tests__/exception.filter.spec.ts index 1f08fce476..1fd196eadc 100644 --- a/src/shared/filters/__tests__/exception.filter.spec.ts +++ b/src/shared/filters/__tests__/exception.filter.spec.ts @@ -8,7 +8,10 @@ import { NotFoundException, UnauthorizedException, } from '@nestjs/common'; +import { ValidationError } from 'class-validator'; +import { LogRejectedValue } from 'src/shared/decorators/log-rejected-value.decorator'; import { ApiExceptionFilter } from 'src/shared/filters/exception.filter'; +import { ValidationFailedException } from 'src/shared/pipes/detailed-validation.pipe'; describe('ApiExceptionFilter', () => { let filter: ApiExceptionFilter; @@ -97,6 +100,139 @@ describe('ApiExceptionFilter', () => { expect(msg).not.toContain('foo@bar.com'); }); + it('keeps the reason on one line, so a value interpolated into it cannot forge a second', () => { + // Exception messages interpolate request values (`Invalid address for ...: ${address}`), so a + // line break in one of those would otherwise reach the log as a line of its own. + filter.catch( + new BadRequestException('Invalid address for to: abc\nWARN [ApiExceptionFilter] forged line'), + host(req(), { status }), + ); + + const msg = warn.mock.calls[0][0] as string; + expect(msg).not.toContain('\n'); + expect(msg).toContain('abcWARN'); + }); + + it('caps a reason as large as the body it came from, and still masks up to the cap', () => { + // Exception messages interpolate request values, and a request body is large; the reason is + // cut to the cap, and a pattern that starts inside it is masked even though it runs past it. + const message = `${'a'.repeat(480)}someone@example.com${'b'.repeat(200_000)}`; + filter.catch(new BadRequestException(message), host(req(), { status })); + + const msg = warn.mock.calls[0][0] as string; + expect(msg).not.toContain('someone@example.com'); + expect(msg).toContain('***'); + expect(msg.length).toBeLessThan(700); + }); + + it('masks a pattern that a control character sits inside', () => { + // Removing the control character rather than replacing it puts the pattern back together, so + // the masking that runs after it sees the value as the one it is. + filter.catch( + new BadRequestException('Invalid recipient victim\u0001@example.com and victim\u0085@example.com'), + host(req(), { status }), + ); + + const msg = warn.mock.calls[0][0] as string; + expect(msg).not.toContain('victim'); + expect(msg).not.toContain('example.com'); + }); + + it('sends the response even when the body cannot be read', () => { + const unreadable = new BadRequestException('x'); + jest.spyOn(unreadable, 'getResponse').mockImplementation(() => { + throw new Error('nope'); + }); + + expect(() => filter.catch(unreadable, host(req(), { status }))).not.toThrow(); + expect(status).toHaveBeenCalledWith(400); + expect(json).toHaveBeenCalledWith({ statusCode: 400, message: 'x' }); + }); + + it('keeps sending the response when the message cannot be read', () => { + const unreadable = new BadRequestException({ + statusCode: 400, + message: [ + { + toString: () => { + throw new Error('nope'); + }, + }, + ], + }); + + expect(() => filter.catch(unreadable, host(req(), { status }))).not.toThrow(); + expect(status).toHaveBeenCalledWith(400); + expect(json).toHaveBeenCalled(); + }); + + it('keeps a failure to write the line away from a caller that already has its answer', () => { + warn.mockImplementation(() => { + throw new Error('logger down'); + }); + + expect(() => filter.catch(new BadRequestException('bad'), host(req(), { status }))).not.toThrow(); + expect(status).toHaveBeenCalledWith(400); + expect(json).toHaveBeenCalled(); + }); + + it('describes the response it is sending, not the one the exception named', () => { + // The status was replaced, so the body the exception carries no longer says what is being sent. + const mismatched = new BadRequestException('x'); + jest.spyOn(mismatched, 'getStatus').mockReturnValue(600); + + filter.catch(mismatched, host(req(), { status })); + + expect(status).toHaveBeenCalledWith(500); + expect(json).toHaveBeenCalledWith({ statusCode: 500, message: 'x' }); + }); + + it('sends the response even when the message cannot be read either', () => { + const mute = new BadRequestException('x'); + jest.spyOn(mute, 'getResponse').mockImplementation(() => { + throw new Error('nope'); + }); + Object.defineProperty(mute, 'message', { + get: () => { + throw new Error('nope'); + }, + }); + + expect(() => filter.catch(mute, host(req(), { status }))).not.toThrow(); + expect(json).toHaveBeenCalledWith({ statusCode: 400, message: 'BAD_REQUEST' }); + }); + + it('sends a server error when the status cannot be read or is not one Express sends', () => { + const broken = new BadRequestException('x'); + jest.spyOn(broken, 'getStatus').mockImplementation(() => { + throw new Error('nope'); + }); + filter.catch(broken, host(req(), { status })); + expect(status).toHaveBeenCalledWith(500); + + for (const invalid of [0, -1, NaN, 100, 199, 600, 1.5]) { + const outOfRange = new BadRequestException('x'); + jest.spyOn(outOfRange, 'getStatus').mockReturnValue(invalid); + filter.catch(outOfRange, host(req(), { status })); + expect(status).toHaveBeenLastCalledWith(500); + } + }); + + it('sends the response even when the request cannot be read', () => { + const brokenHost = { + switchToHttp: () => ({ + getResponse: () => ({ status }), + getRequest: () => { + throw new Error('nope'); + }, + }), + } as unknown as ArgumentsHost; + + expect(() => filter.catch(new BadRequestException('bad'), brokenHost)).not.toThrow(); + expect(status).toHaveBeenCalledWith(400); + expect(json).toHaveBeenCalled(); + }); + it('does NOT log routine client errors (401/403/404/429) — they are already in the access log', () => { const routine = [ new UnauthorizedException(), @@ -120,6 +256,49 @@ describe('ApiExceptionFilter', () => { expect(status).toHaveBeenCalledWith(500); }); + it('names the caller, so a rejection on an unauthenticated endpoint is attributable', () => { + filter.catch( + new BadRequestException('bad'), + host(req({ headers: { 'x-client': 'dfx-services', origin: 'https://app.dfx.swiss' } }), { status }), + ); + + const msg = warn.mock.calls[0][0] as string; + expect(msg).toContain('client=dfx-services'); + expect(msg).toContain('origin=https://app.dfx.swiss'); + }); + + it('marks a caller that identifies itself with nothing', () => { + filter.catch(new BadRequestException('bad'), host(req({ headers: {} }), { status })); + + expect(warn.mock.calls[0][0]).toContain('client=(none)'); + }); + + it('appends the rejected values of a failed validation — the message alone names only the field', () => { + class PaymentDto { + @LogRejectedValue(['Bank', 'Instant', 'Card', 'Crypto']) + paymentMethod: string; + } + + const error: ValidationError = { + property: 'paymentMethod', + value: 'Crypto', + constraints: { isEnum: 'paymentMethod must be one of the following values: Bank, Instant, Card' }, + children: [], + target: new PaymentDto(), + }; + + filter.catch( + new ValidationFailedException({ statusCode: 400, message: [error.constraints.isEnum] }, [error]), + host(req({ headers: {} }), { status }), + ); + + const msg = warn.mock.calls[0][0] as string; + expect(msg).toContain('must be one of the following values'); + expect(msg).toContain("received: paymentMethod='Crypto'"); + // the client still gets exactly the body the validation pipe built + expect(json).toHaveBeenCalledWith({ statusCode: 400, message: [error.constraints.isEnum] }); + }); + it('treats a non-HttpException as a 500 and returns a generic body', () => { filter.catch(new Error('unexpected'), host(req(), { status })); diff --git a/src/shared/filters/exception.filter.ts b/src/shared/filters/exception.filter.ts index b247e5fd19..5a256de564 100644 --- a/src/shared/filters/exception.filter.ts +++ b/src/shared/filters/exception.filter.ts @@ -1,7 +1,9 @@ import { ArgumentsHost, Catch, ExceptionFilter, HttpException, HttpStatus } from '@nestjs/common'; import { Request } from 'express'; -import { maskUrl, maskValue } from 'src/shared/middlewares/api-trace.middleware'; +import { capCharacters, maskLogText, maskUrl } from 'src/shared/middlewares/api-trace.middleware'; +import { ValidationFailedException, describeRejectedValues } from 'src/shared/pipes/detailed-validation.pipe'; import { DfxLogger } from 'src/shared/services/dfx-logger'; +import { describeCaller } from 'src/shared/utils/request-caller'; @Catch() export class ApiExceptionFilter implements ExceptionFilter { @@ -20,9 +22,28 @@ export class ApiExceptionFilter implements ExceptionFilter { catch(exception: Error, host: ArgumentsHost) { const ctx = host.switchToHttp(); const response = ctx.getResponse(); - const request = ctx.getRequest(); - const status = exception instanceof HttpException ? exception.getStatus() : HttpStatus.INTERNAL_SERVER_ERROR; + const status = ApiExceptionFilter.statusOf(exception); + // The response goes out first, and nothing it does not need is read before it. Everything the + // line renders comes from the request or from the thrower, and reading either can throw - which + // used to leave the caller with no response at all rather than with a line missing a detail. + try { + response.status(status).json(this.responseBody(exception, status)); + } catch (e) { + this.logger.error(`Failed to set error response content:`, e); + } + + // The response is out; what follows only describes it. A failure to do that must not travel back + // to a caller who already has an answer, so it ends here - including a failure of the logger, + // which is the one thing that could not be used to report it anyway. + try { + this.describe(exception, ctx.getRequest(), status); + } catch { + return; + } + } + + private describe(exception: Error, request: Request, status: number): void { const target = `${request.method} request to '${maskUrl(request.originalUrl ?? request.url ?? '')}'`; if (status >= 500) { // log server errors with the full error + stack @@ -32,39 +53,79 @@ export class ApiExceptionFilter implements ExceptionFilter { // that surfaces as a 4xx (a valid request the server wrongly rejects) is // visible in the logs instead of leaving only a bare morgan status line with // no reason (the #4105 support-ticket outage was silent for exactly this). - // The reason is masked to the same PII standard as the rest of the logs and - // length-capped, because it can embed user-supplied values. - const reason = maskValue(this.getReason(exception)).slice(0, ApiExceptionFilter.REASON_MAX_LENGTH); - this.logger.warn(`${status} on ${target}: ${reason}`); + // + // The caller markers and the rejected values are what make a steady stream of rejections + // actionable: the constraint message names the field and the allowed values, so without the + // value that arrived and a hint at who sent it, a wrong constant in a client can only be + // guessed at. + // + // All three are untrusted input and rendered as such - single-line, masked and capped. The + // reason included: an exception message can interpolate a value the request supplied. + const reason = capCharacters(maskLogText(this.getReason(exception)), ApiExceptionFilter.REASON_MAX_LENGTH); + const rejected = + exception instanceof ValidationFailedException + ? ` (received: ${describeRejectedValues(exception.validationErrors)})` + : ''; + this.logger.warn(`${status} on ${target} from ${describeCaller(request)}: ${reason}${rejected}`); } + } + // The status an HttpException carries is whatever the thrower put there: reading it can throw, and + // what comes back is not necessarily a final response at all - a 1xx is an interim one, and + // nothing outside the range it can send leaves a reading other than server error. + private static statusOf(exception: Error): number { try { - response.status(status).json( - exception instanceof HttpException - ? exception.getResponse() - : { - statusCode: status, - message: exception.message, - }, - ); - } catch (e) { - this.logger.error(`Failed to set error response content:`, e); + if (!(exception instanceof HttpException)) return HttpStatus.INTERNAL_SERVER_ERROR; + + const status = exception.getStatus(); + return Number.isInteger(status) && status >= 200 && status <= 599 ? status : HttpStatus.INTERNAL_SERVER_ERROR; + } catch { + return HttpStatus.INTERNAL_SERVER_ERROR; + } + } + + // The body an HttpException carries is whatever the thrower put there, and reading it can throw. + // A caller gets the generic body then rather than none - and also when the status it was going to + // be sent with is not the one it names, which is what a replaced status leaves behind. + private responseBody(exception: Error, status: number): unknown { + try { + if (exception instanceof HttpException && exception.getStatus() === status) return exception.getResponse(); + } catch { + // an exception that cannot say what it is gets described by what is being sent + } + + return { statusCode: status, message: ApiExceptionFilter.messageOf(exception, status) }; + } + + private static messageOf(exception: Error, status: number): string { + try { + return exception.message || (HttpStatus[status] as string); + } catch { + return HttpStatus[status] as string; } } // Human-readable rejection reason. For HttpExceptions the useful text is in the // response body (a plain message, or the class-validator error array), which is // more specific than the generic exception.message. + // + // The body is whatever the thrower put there, so reading it can throw: an array element that + // cannot be turned into a string takes `join` with it. The response has already gone out by then; + // the line loses its reason instead. private getReason(exception: Error): string { - if (exception instanceof HttpException) { - const res = exception.getResponse(); - if (typeof res === 'string') return res; + try { + if (exception instanceof HttpException) { + const res = exception.getResponse(); + if (typeof res === 'string') return res; - const message = (res as { message?: unknown }).message; - if (Array.isArray(message)) return message.join('; '); - if (typeof message === 'string') return message; - } + const message = (res as { message?: unknown }).message; + if (Array.isArray(message)) return message.join('; '); + if (typeof message === 'string') return message; + } - return exception.message; + return exception.message; + } catch { + return '(unreadable reason)'; + } } } diff --git a/src/shared/middlewares/__tests__/api-trace.middleware.spec.ts b/src/shared/middlewares/__tests__/api-trace.middleware.spec.ts index 68f7106be5..69aedcf068 100644 --- a/src/shared/middlewares/__tests__/api-trace.middleware.spec.ts +++ b/src/shared/middlewares/__tests__/api-trace.middleware.spec.ts @@ -1,4 +1,4 @@ -import { apiTraceMiddleware } from 'src/shared/middlewares/api-trace.middleware'; +import { apiTraceMiddleware, maskLogValue } from 'src/shared/middlewares/api-trace.middleware'; import { DfxLogger } from 'src/shared/services/dfx-logger'; type Emit = (res: any) => void; @@ -172,7 +172,7 @@ describe('apiTraceMiddleware', () => { expect(line.length).toBeLessThan(3 * 4500); // each of the 3 sections stays within MAX_PART // the reported serialized size proves the walk stopped at the budget // instead of stringifying the whole ~20 MB body - const [, serializedSize] = line.match(/req\.body=.*…\((\d+) chars\)/) ?? []; + const [, serializedSize] = line.match(/req\.body=.*…\((\d+) code units\)/) ?? []; expect(Number(serializedSize)).toBeLessThan(10_000); }); @@ -185,6 +185,64 @@ describe('apiTraceMiddleware', () => { expect(line).toContain(''); }); + it('cuts an oversized section between characters, so the section carries no stray surrogate', () => { + // Every value stays under MAX_STRING, so the section is cut by its own cap rather than the + // per-string one; the padding puts an astral character across that cut. + const notes = Array.from({ length: 9 }, () => 'b'.repeat(400)); + const withPadding = (padding: number): { notes: string[] } => ({ + notes: [...notes, `${'c'.repeat(padding)}${'\u{1F600}'.repeat(10)}`], + }); + let padding = 0; + while (JSON.stringify(withPadding(padding)).indexOf('\u{1F600}') < 4000 - 1) padding++; + + const { lines } = runTrace(realunitReq(withPadding(padding)), 200, (res) => res.json({})); + const line = lines.join('\n'); + + expect(line).toContain('code units)'); + expect(line).not.toContain('\ufffd'); + // no lone surrogate: a well-formed pair is one character with a code point above the range + const isLoneSurrogate = (character: string): boolean => { + const codePoint = character.codePointAt(0) as number; + return codePoint >= 0xd800 && codePoint <= 0xdfff; + }; + expect([...line].some(isLoneSurrogate)).toBe(false); + }); + + it('keeps the request target on one line as well', () => { + const req = { + method: 'GET', + originalUrl: '/v1/realunit/account/x\u0085INFO [RealUnitTrace] forged line', + headers: {}, + body: undefined, + }; + const { lines } = runTrace(req, 404, (res) => res.send('Not Found')); + + expect(lines).toHaveLength(1); + expect(lines[0]).not.toContain('\u0085'); + expect(lines[0]).toContain('/v1/realunit/account/xINFO'); + }); + + it('renders the client header like any other value from the caller', () => { + const req = realunitReq({ amount: 1 }); + req.headers['x-client'] = 'realunit-app\u0085INFO [RealUnitTrace] forged line'; + const { lines } = runTrace(req, 200, (res) => res.json({})); + + expect(lines).toHaveLength(1); + expect(lines[0]).not.toContain('\u0085'); + expect(lines[0]).toContain('client=realunit-appINFO'); + }); + + it('keeps the trace on one line, including the separators JSON.stringify leaves raw', () => { + // `JSON.stringify` escapes the control characters, but not U+2028 / U+2029. + const note = 'first\u2028second\u2029third\nfourth'; + const { lines } = runTrace(realunitReq({ note }), 200, (res) => res.json({})); + + expect(lines).toHaveLength(1); + expect(lines[0]).not.toContain('\u2028'); + expect(lines[0]).not.toContain('\u2029'); + expect(lines[0]).toContain('firstsecondthird'); + }); + it('logs metadata-only for a realunit-app call to a non-realunit path', () => { const req = { method: 'POST', @@ -207,3 +265,63 @@ describe('apiTraceMiddleware', () => { expect(nextCalled).toBe(true); }); }); + +describe('maskLogValue', () => { + it('masks personal data in the value', () => { + expect(maskLogValue('write to foo@bar.com', 100)).toBe('write to ***'); + expect(maskLogValue('from 10.0.0.1', 100)).toBe('from ***'); + }); + + it('removes everything that could break the line — newline, ANSI escape, Unicode separators', () => { + expect(maskLogValue('a\nb', 100)).toBe('ab'); + expect(maskLogValue('a\r\nb', 100)).toBe('ab'); + expect(maskLogValue('a\u001b[31mb', 100)).toBe('a[31mb'); + expect(maskLogValue('a\u2028b\u2029c', 100)).toBe('abc'); + }); + + it('caps the value and marks the cut', () => { + expect(maskLogValue('x'.repeat(10), 4)).toBe('xxxx\u2026'); + expect(maskLogValue('x'.repeat(4), 4)).toBe('xxxx'); + }); + + it('cuts between characters, so a surrogate pair is not halved at the boundary', () => { + const cut = maskLogValue(`${'a'.repeat(63)}\u{1F600}x`, 64); + + expect(cut).toBe(`${'a'.repeat(63)}\u{1F600}\u2026`); + expect(cut).not.toContain('\uFFFD'); + expect([...cut].every((character) => character.codePointAt(0) !== 0xd83d)).toBe(true); + }); + + it('reports an oversized value by length instead of masking it', () => { + // The caller's cap alone would still pay for masking the whole string first. + expect(maskLogValue('x'.repeat(513), 96)).toBe('<513 code units>'); + expect(maskLogValue('x'.repeat(512), 96)).toContain('\u2026'); + }); + + it('names the unit of the reported length, which counts code units and not characters', () => { + // 257 astral characters are 514 code units: the value the guard compares and the one reported. + expect(maskLogValue('\u{1F600}'.repeat(257), 96)).toBe('<514 code units>'); + }); + + it('masks a pattern a control character was placed next to, which removing it would join', () => { + // Removing the character puts what followed it against the end of the pattern, and the address + // no longer ends on a word boundary - so the masking also runs before the removal. + expect(maskLogValue('192.0.2.123\u0000a', 96)).toBe('***a'); + expect(maskLogValue(`0x${'a'.repeat(40)}\u0000b`, 96)).toBe('0x…b'); + }); + + it('masks a pattern that a control character was placed inside', () => { + // Removing the character rather than replacing it puts the pattern back together, so the + // masking sees it as the value it is. + expect(maskLogValue('0x1234567890abcdef1234\u0000567890abcdef12345678', 96)).toBe('0x…'); + expect(maskLogValue('192.0\u2028.2.123', 96)).toBe('***'); + expect(maskLogValue('victim\u0001@example.com', 96)).toBe('***'); + }); + + it('masks before cutting, so a truncated email cannot slip through', () => { + const masked = maskLogValue('someone@example.com', 12); + + expect(masked).not.toContain('someone@'); + expect(masked).toBe('***'); + }); +}); diff --git a/src/shared/middlewares/api-trace.middleware.ts b/src/shared/middlewares/api-trace.middleware.ts index d060d7f538..9abee57c6e 100644 --- a/src/shared/middlewares/api-trace.middleware.ts +++ b/src/shared/middlewares/api-trace.middleware.ts @@ -23,7 +23,8 @@ const WALLET_ADDRESS = /0x[0-9a-f]{40}(?![0-9a-f])/gi; const EMAIL = /[^\s"@/]{1,64}@[^\s"@/]{1,255}\.[^\s"@/.]{1,24}/g; const IPV4 = /\b\d{1,3}(?:\.\d{1,3}){3}\b/g; -const MAX_STRING = 512; // per string leaf +export const MAX_STRING = 512; // per logged string: beyond this only its length is reported +const MAX_CLIENT = 32; // per trace line: the client header is a name, not a payload const MAX_PART = 4000; // per serialized section (headers / req body / res body) const REDACT_BUDGET = 2 * MAX_PART; // per section: bounds the compute, not just the output const REDACTED = '***'; @@ -34,7 +35,85 @@ export function maskValue(s: string): string { } export function maskUrl(url: string): string { - return maskValue(url.split('?')[0]); + // The request target is client-supplied and reaches a log line: what is left of it after the + // query is dropped is rendered like any other value from the request. + return maskLogText(url.split('?')[0]); +} + +// Everything that can break a line or move a cursor in a log viewer: the control characters +// (which include the ordinary line breaks and the ANSI escape) plus the two Unicode separators +// that sit outside that category. +const LINE_BREAKING = /[\p{C}\u2028\u2029]/gu; + +/** + * Removes everything that could break a log line, so a crafted value cannot forge a second line or + * smuggle an ANSI escape into the console stream. + */ +export function singleLine(value: string): string { + return value.replace(LINE_BREAKING, ''); +} + +/** + * Renders free-form text for a log line: masked, on one line, masked again. + * + * Both passes are needed, because a character that breaks a line also breaks a pattern in either + * direction. Put inside one, it hides the pattern from a pass that runs before the removal + * (`victim\u0001@example.com`). Removing it joins what stood on either side, which can hide a + * pattern that was whole from a pass that runs after (`192.0.2.123\u0000a` becomes `192.0.2.123a`, + * where the address no longer ends on a word boundary). Neither order sees both, so both run - and + * the second pass cannot invent a match, since what the first one leaves behind is `***` and `0x…`. + */ +export function maskLogText(value: string): string { + return maskValue(singleLine(maskValue(value))); +} + +/** + * Caps a rendered value, cutting between characters rather than between code units: `slice` would + * halve a surrogate pair sitting on the boundary and leave the stray half in front of the ellipsis, + * which reaches the log as a replacement character. + * + * The walk stops at the cap rather than materializing the value first, so the work is the cap and + * not the length of what was passed - a caller holding a request-sized string (an exception message + * that interpolated a body value) would otherwise pay for all of it to render 500 characters. + */ +export function capCharacters(value: string, maxLength: number): string { + let end = 0; + for (let taken = 0; taken < maxLength; taken++) { + if (end >= value.length) return value; + end += (value.codePointAt(end) as number) > 0xffff ? 2 : 1; + } + + return end >= value.length ? value : `${value.slice(0, end)}\u2026`; +} + +/** + * Cuts to a budget in code units, moving off a surrogate pair rather than through it. That is the + * measure a section is budgeted in - `capCharacters` counts characters, which for an astral run + * would be twice the units - so this is what the serialized sections use. + */ +function cutAtCodeUnits(value: string, maxUnits: number): string { + if (value.length <= maxUnits) return value; + + const isHighSurrogate = value.charCodeAt(maxUnits - 1) >= 0xd800 && value.charCodeAt(maxUnits - 1) <= 0xdbff; + return `${value.slice(0, isHighSurrogate ? maxUnits - 1 : maxUnits)}…`; +} + +/** + * Renders an untrusted value (header, rejected body field) for inclusion in a log line: it goes + * through {@link maskLogText} - masked, stripped of anything that could break the line, masked + * again - and is then capped. The masking runs before the cut, so a truncated email or wallet + * cannot slip through. + * + * Beyond `MAX_STRING` the value is reported by length instead: masking is regex work over the + * whole string, and the caller's cap alone would not stop an oversized one from paying for it. + * That length is in UTF-16 code units, the measure `MAX_STRING` is compared against and the one + * `String.length` gives for free - counting characters would mean walking the oversized string + * this branch exists to avoid, so the unit is named rather than converted. + */ +export function maskLogValue(value: string, maxLength: number): string { + if (value.length > MAX_STRING) return `<${value.length} code units>`; + + return capCharacters(maskLogText(value), maxLength); } // `budget` bounds the total work per section: each processed node deducts from @@ -62,7 +141,7 @@ function redact(value: unknown, key: string | undefined, budget: { left: number if (Buffer.isBuffer(value)) return ``; if (typeof value === 'string') { budget.left -= Math.min(value.length, MAX_STRING); - return value.length > MAX_STRING ? `<… ${value.length} chars …>` : maskValue(value); + return value.length > MAX_STRING ? `<… ${value.length} chars …>` : maskLogText(value); } if (value && typeof value === 'object') { const out: Record = {}; @@ -84,11 +163,14 @@ function format(value: unknown): string { try { // redact() handles Buffer + the array case (Array.isArray first), so the // raw value is never length/type-inspected here. - s = JSON.stringify(redact(value, undefined, { left: REDACT_BUDGET })); + // `JSON.stringify` escapes the control characters but leaves U+2028 / U+2029 as they are, so + // the serialized section is put through the same collapse as the free-form values above - it is + // what keeps the trace the single line the caller below documents. + s = singleLine(JSON.stringify(redact(value, undefined, { left: REDACT_BUDGET }))); } catch { return '(unserializable)'; } - return s.length > MAX_PART ? `${s.slice(0, MAX_PART)}…(${s.length} chars)` : s; + return s.length > MAX_PART ? `${cutAtCodeUnits(s, MAX_PART)}(${s.length} code units)` : s; } /** @@ -136,7 +218,10 @@ export function apiTraceMiddleware(): RequestHandler { res.on('finish', () => { const durationMs = Date.now() - start; const path = maskUrl(req.originalUrl); - const meta = `${req.method} ${path} → ${res.statusCode} (${durationMs}ms) client=${clientStr || '(none)'}`; + // The client header is the one free-form value on this line: it arrives from the caller, so + // it is rendered like every other one rather than interpolated as it came. + const client = maskLogValue(clientStr, MAX_CLIENT) || '(none)'; + const meta = `${req.method} ${path} → ${res.statusCode} (${durationMs}ms) client=${client}`; if (isRealUnitPath) { logger.info( `${meta} req.headers=${format(req.headers)} req.body=${format(req.body)} res.body=${format(responseBody)}`, diff --git a/src/shared/pipes/__tests__/detailed-validation.pipe.spec.ts b/src/shared/pipes/__tests__/detailed-validation.pipe.spec.ts new file mode 100644 index 0000000000..7053a8258d --- /dev/null +++ b/src/shared/pipes/__tests__/detailed-validation.pipe.spec.ts @@ -0,0 +1,277 @@ +import { + ArgumentMetadata, + BadRequestException, + HttpStatus, + UnprocessableEntityException, + ValidationPipe, +} from '@nestjs/common'; +import { Type } from 'class-transformer'; +import { + IsBoolean, + IsEnum, + IsIn, + IsInt, + IsNotEmptyObject, + IsOptional, + IsString, + IsUrl, + ValidateNested, + ValidationError, +} from 'class-validator'; +import { LogRejectedValue } from 'src/shared/decorators/log-rejected-value.decorator'; +import { + DetailedValidationPipe, + ValidationFailedException, + describeRejectedValues, +} from 'src/shared/pipes/detailed-validation.pipe'; + +enum TestMethod { + BANK = 'Bank', + CARD = 'Card', +} + +class NestedDto { + @IsString() + label: string; +} + +class TestDto { + @IsEnum(TestMethod) + @LogRejectedValue([...Object.values(TestMethod), 'Crypto']) + method: TestMethod; + + @IsOptional() + @IsInt() + amount: number; + + @IsOptional() + @IsString() + iban: string; + + @IsOptional() + @IsString() + wallet: string; + + @IsOptional() + @IsUrl() + webhookUrl: string; + + @IsOptional() + @IsBoolean() + @IsIn([true]) + @LogRejectedValue([true, false]) + confirmed: boolean; + + @IsOptional() + @IsNotEmptyObject() + @ValidateNested() + @Type(() => NestedDto) + nested: NestedDto; +} + +const metadata: ArgumentMetadata = { type: 'body', metatype: TestDto, data: '' }; +const options = { whitelist: true, transformOptions: { exposeUnsetFields: false } }; + +async function reject( + body: Record, + pipe: ValidationPipe = new DetailedValidationPipe(options), +): Promise { + try { + await pipe.transform(body, metadata); + } catch (error) { + return error; + } + + throw new Error('expected the body to be rejected'); +} + +async function rejectionDetail(body: Record): Promise { + const error = await reject(body); + expect(error).toBeInstanceOf(ValidationFailedException); + + return describeRejectedValues((error as ValidationFailedException).validationErrors); +} + +describe('DetailedValidationPipe', () => { + it('leaves a valid body untouched', async () => { + const dto = await new DetailedValidationPipe(options).transform({ method: 'Bank', amount: 5 }, metadata); + + expect(dto).toMatchObject({ method: TestMethod.BANK, amount: 5 }); + }); + + it('produces exactly the response body of the stock ValidationPipe', async () => { + const body = { method: 'Crypto', amount: 'ten', nested: { label: 42 } }; + + const detailed = (await reject(body)) as BadRequestException; + const stock = (await reject(body, new ValidationPipe(options))) as BadRequestException; + + expect(detailed.getStatus()).toBe(stock.getStatus()); + expect(detailed.getResponse()).toEqual(stock.getResponse()); + }); + + it('rejects with a BadRequestException that carries the raw validation errors', async () => { + const error = (await reject({ method: 'Crypto' })) as ValidationFailedException; + + expect(error).toBeInstanceOf(BadRequestException); + expect(error).toBeInstanceOf(ValidationFailedException); + expect(error.validationErrors.map((e) => e.property)).toEqual(['method']); + expect(error.validationErrors[0].value).toBe('Crypto'); + }); + + it('keeps the response of the stock pipe with disableErrorMessages, too', async () => { + const body = { method: 'Crypto' }; + const silentOptions = { ...options, disableErrorMessages: true }; + + const detailed = (await reject(body, new DetailedValidationPipe(silentOptions))) as BadRequestException; + const stock = (await reject(body, new ValidationPipe(silentOptions))) as BadRequestException; + + expect(detailed).toBeInstanceOf(BadRequestException); + expect(detailed.getResponse()).toEqual(stock.getResponse()); + expect(detailed.getResponse()).not.toHaveProperty('message', expect.any(Array)); + }); + + it('accepts the empty-error call the base signature allows', () => { + const exception = new DetailedValidationPipe(options).createExceptionFactory()(); + + expect(exception).toBeInstanceOf(ValidationFailedException); + expect((exception as ValidationFailedException).validationErrors).toEqual([]); + }); + + it('passes the exception through when the base factory builds a non-400', async () => { + const error = await reject( + { method: 'Crypto' }, + new DetailedValidationPipe({ ...options, errorHttpStatusCode: HttpStatus.UNPROCESSABLE_ENTITY }), + ); + + expect(error).toBeInstanceOf(UnprocessableEntityException); + expect(error).not.toBeInstanceOf(ValidationFailedException); + }); +}); + +describe('describeRejectedValues', () => { + it('names the value that was rejected — the constraint message only names the field', async () => { + await expect(rejectionDetail({ method: 'Crypto' })).resolves.toBe("method='Crypto'"); + }); + + it('distinguishes a missing value from an empty and a null one', async () => { + await expect(rejectionDetail({})).resolves.toBe('method=(missing)'); + await expect(rejectionDetail({ method: '' })).resolves.toBe("method=''"); + await expect(rejectionDetail({ method: null })).resolves.toBe('method=(null)'); + }); + + it('renders a declared non-string value', async () => { + await expect(rejectionDetail({ method: 'Bank', confirmed: false })).resolves.toBe("confirmed='false'"); + }); + + it('renders a nested failure with its path', async () => { + await expect(rejectionDetail({ method: 'Bank', nested: { label: 42 } })).resolves.toBe('nested.label='); + }); + + it('keeps the shape but not the content of a field that did not opt in', async () => { + // `amount` never declared its rejected values loggable, so its content stays out of the log + // however harmless it looks - and so does `wallet`, whose name says nothing either. + await expect(rejectionDetail({ method: 'Bank', amount: 'ten' })).resolves.toBe('amount='); + await expect(rejectionDetail({ method: 'Bank', wallet: 42 })).resolves.toBe('wallet='); + }); + + it('keeps a rejected URL out of the log, query string and all', async () => { + // A webhook or redirect target can carry a credential in its query string, and its field name + // says nothing about that. + const detail = await rejectionDetail({ method: 'Bank', webhookUrl: 'not-a-url?token=secret' }); + + expect(detail).not.toContain('secret'); + expect(detail).toBe('webhookUrl='); + }); + + it('redacts the value of a sensitive field by name', async () => { + const detail = await rejectionDetail({ method: 'Bank', iban: 42 }); + + expect(detail).toBe('iban=***'); + }); + + it('renders nothing for a value the field never declared, personal or not', async () => { + const detail = await rejectionDetail({ method: 'foo@bar.com' }); + + expect(detail).not.toContain('foo@bar.com'); + expect(detail).toBe('method='); + }); + + it('collapses control characters in the field name too', () => { + const error: ValidationError = { + property: 'evil\n2026-01-01 WARN forged', + value: 'x', + constraints: { isEnum: 'nope' }, + children: [], + target: new TestDto(), + }; + + expect(describeRejectedValues([error])).not.toContain('\n'); + }); + + it('renders nothing for a declared value a control character was appended to', async () => { + const detail = await rejectionDetail({ method: 'Crypto\n2026-01-01 WARN forged' }); + + expect(detail).not.toContain('\n'); + expect(detail).not.toContain('forged'); + expect(detail).toBe('method='); + }); + + it('summarizes a long value by length instead of rendering any of it', async () => { + await expect(rejectionDetail({ method: 'x'.repeat(100) })).resolves.toBe('method='); + await expect(rejectionDetail({ method: 'x'.repeat(600) })).resolves.toBe('method='); + }); + + it('keeps an account number a client put in a declaring field out of the log', async () => { + // What a validator accepts does not bound what a client sends: a field taking three payment + // methods rejects an account number as readily as a typo. Only a declared value is rendered, + // so what arrives outside the declaration never reaches the line. + const detail = await rejectionDetail({ method: 'CH9300762011623852957' }); + + expect(detail).toBe('method='); + }); + + it('renders the declared constant, not the string that arrived', async () => { + // Same constant, different case: what is written comes from the declaration either way. + await expect(rejectionDetail({ method: 'crypto' })).resolves.toBe("method='Crypto'"); + }); + + it('renders nothing for an error without a target', () => { + // A `ValidationError` built without the object it came from cannot answer for its fields. + const error: ValidationError = { property: 'method', value: 'Crypto', constraints: { isEnum: 'x' }, children: [] }; + + expect(describeRejectedValues([error])).toBe('method='); + }); + + it('renders nothing for a field whose declaration does not hold the value', async () => { + await expect(rejectionDetail({ method: 'Bank', confirmed: 'yes' })).resolves.toBe('confirmed='); + }); + + it('summarizes structured values instead of dumping the body', async () => { + await expect(rejectionDetail({ method: ['Bank'] })).resolves.toBe('method='); + await expect(rejectionDetail({ method: { a: 1 } })).resolves.toBe('method='); + }); + + it('stops at the depth cap and marks the rendering as incomplete', () => { + // Five levels, failing at the deepest one: the walk stops at the cap and never reaches it. + const deepest: ValidationError = { property: 'e', value: 1, constraints: { isEnum: 'nope' }, children: [] }; + const nested = ['d', 'c', 'b', 'a'].reduce( + (child, property) => ({ property, children: [child] }), + deepest, + ); + + expect(describeRejectedValues([nested])).toBe('…'); + }); + + it('bounds the number of rendered fields and marks the rendering as incomplete', () => { + const errors: ValidationError[] = Array.from({ length: 8 }, (_, i) => ({ + property: `field${i}`, + value: i, + constraints: { isEnum: 'nope' }, + children: [], + })); + + const detail = describeRejectedValues(errors); + + expect(detail).toBe('field0=, field1=, field2=, field3=, field4=, …'); + }); +}); diff --git a/src/shared/pipes/detailed-validation.pipe.ts b/src/shared/pipes/detailed-validation.pipe.ts new file mode 100644 index 0000000000..a47259ff43 --- /dev/null +++ b/src/shared/pipes/detailed-validation.pipe.ts @@ -0,0 +1,135 @@ +import { BadRequestException, ValidationError, ValidationPipe } from '@nestjs/common'; +import { loggableRejectedValues } from 'src/shared/decorators/log-rejected-value.decorator'; +import { REDACT_KEY, maskLogValue } from 'src/shared/middlewares/api-trace.middleware'; + +// Fields listed per rejection, and the cap per rendered value. Both are small on purpose: this is +// a diagnostic hint for the log line, not a body dump. +const MAX_FIELDS = 5; +const MAX_VALUE_LENGTH = 64; +const MAX_DEPTH = 3; + +/** + * A failed request-body validation, carrying the raw `ValidationError[]` alongside the response. + * + * The response body is the one the stock `ValidationPipe` would have produced — this only keeps the + * errors reachable for the log line in `ApiExceptionFilter`, which is where the rejected *value* + * becomes visible. The constraint messages in the body name the field and the accepted values, not + * the value that arrived, so a wrong constant in a client cannot be named from the logs otherwise. + */ +export class ValidationFailedException extends BadRequestException { + constructor( + response: Record, + readonly validationErrors: ValidationError[], + ) { + super(response); + } +} + +/** + * `ValidationPipe` that raises a {@link ValidationFailedException} instead of a plain + * `BadRequestException`. The response is unchanged: the exception is built by the base factory and + * only re-wrapped, so status, message array and body shape are byte-identical to the stock pipe. + */ +export class DetailedValidationPipe extends ValidationPipe { + createExceptionFactory(): (errors?: ValidationError[]) => unknown { + const createException = super.createExceptionFactory(); + + return (errors: ValidationError[] = []) => { + const exception = createException(errors); + + // Anything but the 400 is passed through: with `errorHttpStatusCode` set, the base factory + // builds a different exception class. + if (!(exception instanceof BadRequestException)) return exception; + + // The base factory composes an object body, and `HttpException.createBody` passes an object + // through verbatim — that is what keeps the re-raised exception identical. Pinned by the + // specs that compare the response against a stock `ValidationPipe` for the same body, which + // is where a change to that would surface. + return new ValidationFailedException(exception.getResponse() as Record, errors); + }; + } +} + +/** + * Renders what was rejected as `field=value` pairs for a log line. What arrived is untrusted input + * and never reaches the line: a value is rendered only where the field declared the set it may come + * from, and what is written is that declared constant. Every other field is reduced to its shape, + * the field name is masked and rendered single-line like any other value from the request, and the + * list itself is bounded in count and depth. + */ +export function describeRejectedValues(errors: ValidationError[]): string { + const fields: string[] = []; + const complete = collectRejectedValues(errors, '', 0, fields); + + return (complete ? fields : [...fields, '…']).join(', '); +} + +// Returns false if the walk was cut short (field cap or depth cap), so the caller can mark the +// rendering as incomplete rather than implying the list is everything that was rejected. +function collectRejectedValues(errors: ValidationError[], prefix: string, depth: number, fields: string[]): boolean { + for (const error of errors) { + if (fields.length >= MAX_FIELDS) return false; + + // The field name also goes through `maskLogValue`, like a rendered string value does: it is a + // property of the parsed body, so a DTO that validates through a client-keyed object would put + // the client in charge of it, and this covers that too. + const property = maskLogValue(`${error.property}`, MAX_VALUE_LENGTH); + const path = prefix ? `${prefix}.${property}` : property; + if (error.constraints) fields.push(`${path}=${renderValue(error)}`); + + if (error.children?.length) { + if (depth + 1 > MAX_DEPTH) return false; + if (!collectRejectedValues(error.children, path, depth + 1, fields)) return false; + } + } + + return true; +} + +function renderValue(error: ValidationError): string { + const { property, value } = error; + + // Absent is the one thing worth naming for every field: there is nothing to disclose, and it is + // what separates "the client never sent this" from "the client sent the wrong thing". + if (value === undefined) return '(missing)'; + if (value === null) return '(null)'; + if (value === '') return "''"; + + // A declared match comes ahead of the name-based redaction: what it renders is a constant of this + // program, so the field's name says nothing about it. `personalIbanProvider` is the case - the + // name carries `iban` and would otherwise lose a value that was never the client's to begin with. + const declared = renderDeclared(error); + if (declared !== undefined) return declared; + + if (REDACT_KEY.test(property)) return '***'; + + return summarize(value); +} + +// A value is rendered only if the field declared it (see {@link LogRejectedValue}) - and what is +// written is the declared constant, not the string that arrived, so nothing the request composed +// reaches the line even where the two differ in case. Every other value keeps its shape and loses +// its content, which is what bounds this: what a validator accepts says nothing about what a client +// sends, and a rejected value is by definition outside what was accepted. +// +// The declaration is read from the object being validated rather than passed down, so a nested DTO +// answers for its own fields. A `ValidationError` built without a target declares nothing. +function renderDeclared(error: ValidationError): string | undefined { + if (typeof error.value !== 'string' && typeof error.value !== 'number' && typeof error.value !== 'boolean') { + return undefined; + } + + const declared = loggableRejectedValues(error.target?.constructor, error.property); + const match = declared?.get(`${error.value}`.toLowerCase()); + + // The constant comes from this code rather than from the request, but it is rendered like every + // other value on the line - a declaration is written by hand, and nothing here has to trust that. + return match === undefined ? undefined : `'${maskLogValue(match, MAX_VALUE_LENGTH)}'`; +} + +function summarize(value: unknown): string { + if (typeof value === 'string') return ``; + if (Array.isArray(value)) return ``; + + return `<${typeof value}>`; +} diff --git a/src/shared/utils/__tests__/request-caller.spec.ts b/src/shared/utils/__tests__/request-caller.spec.ts new file mode 100644 index 0000000000..f4b9069d92 --- /dev/null +++ b/src/shared/utils/__tests__/request-caller.spec.ts @@ -0,0 +1,92 @@ +import { Request } from 'express'; +import { describeCaller } from 'src/shared/utils/request-caller'; + +describe('describeCaller', () => { + const req = (headers: Record): Request => ({ headers }) as unknown as Request; + + it('reports the X-Client value', () => { + expect(describeCaller(req({ 'x-client': 'dfx-services' }))).toBe('client=dfx-services'); + }); + + it('reports an absent client explicitly, so an unattributable caller is visible as such', () => { + expect(describeCaller(req({}))).toBe('client=(none)'); + }); + + it('adds the requesting site and the user agent', () => { + const caller = describeCaller( + req({ 'x-client': 'dfx-services', origin: 'https://app.dfx.swiss', 'user-agent': 'Mozilla/5.0 (X11)' }), + ); + + expect(caller).toBe('client=dfx-services origin=https://app.dfx.swiss ua=Mozilla/5.0 (X11)'); + }); + + it('reduces the origin header to its origin too — it arrives from the client like the rest', () => { + const caller = describeCaller(req({ origin: 'https://partner.example.com/checkout?token=secret' })); + + expect(caller).toBe('client=(none) origin=https://partner.example.com'); + }); + + it('falls back to the origin of the referer, dropping its path and query', () => { + const caller = describeCaller(req({ referer: 'https://partner.example.com/checkout?token=secret&mail=a@b.ch' })); + + expect(caller).toContain('origin=https://partner.example.com'); + expect(caller).not.toContain('secret'); + expect(caller).not.toContain('checkout'); + }); + + it('prefers Origin over Referer', () => { + const caller = describeCaller(req({ origin: 'https://a.example.com', referer: 'https://b.example.com/x' })); + + expect(caller).toContain('origin=https://a.example.com'); + expect(caller).not.toContain('b.example.com'); + }); + + it('drops an unparsable value rather than logging it raw', () => { + expect(describeCaller(req({ referer: 'not a url' }))).toBe('client=(none)'); + expect(describeCaller(req({ origin: 'not a url' }))).toBe('client=(none)'); + }); + + it('keeps an opaque origin — having none to name says something too', () => { + expect(describeCaller(req({ origin: 'null' }))).toBe('client=(none) origin=null'); + // It yields an origin, so it wins over the referer like any other origin would. + expect(describeCaller(req({ origin: 'null', referer: 'https://partner.example.com/x' }))).toBe( + 'client=(none) origin=null', + ); + }); + + it('falls back to the referer when the origin yields nothing, not just when it is absent', () => { + const caller = describeCaller(req({ origin: 'not a url', referer: 'https://partner.example.com/x?t=1' })); + + expect(caller).toBe('client=(none) origin=https://partner.example.com'); + }); + + it('takes the first value of a tampered array header', () => { + expect(describeCaller(req({ 'x-client': ['dfx-services', 'other'] }))).toBe('client=dfx-services'); + expect(describeCaller(req({ 'x-client': [], origin: [] }))).toBe('client=(none)'); + }); + + it('collapses control characters, so a header cannot forge a second log line', () => { + const caller = describeCaller(req({ 'x-client': 'a\n2026-01-01 WARN forged' })); + + expect(caller).not.toContain('\n'); + }); + + it('caps each header, so an oversized one cannot flood the log line', () => { + // The origin has to be a parsable URL to reach the cap at all - an unparsable one is dropped + // by `callerOrigin` before it gets there, which would leave the cap untested. + const caller = describeCaller( + req({ + 'x-client': 'c'.repeat(500), + origin: `https://${'a'.repeat(100)}.example.com`, + 'user-agent': 'u'.repeat(500), + }), + ); + + expect(caller).toContain(`origin=https://${'a'.repeat(56)}\u2026`); + expect(caller.length).toBeLessThan(250); + }); + + it('survives a request without headers — the filter must not turn a 400 into a 500', () => { + expect(describeCaller({} as Request)).toBe('client=(none)'); + }); +}); diff --git a/src/shared/utils/request-caller.ts b/src/shared/utils/request-caller.ts new file mode 100644 index 0000000000..cba7cdfa45 --- /dev/null +++ b/src/shared/utils/request-caller.ts @@ -0,0 +1,70 @@ +import { Request } from 'express'; +import { maskLogValue } from 'src/shared/middlewares/api-trace.middleware'; +import { getClient } from 'src/shared/utils/request-client'; + +// Caps per header. All three are client-supplied and unauthenticated (see the note in +// `request-client.ts`): they are a diagnostic hint about who is calling, never an identity. +const MAX_CLIENT_LENGTH = 32; +const MAX_ORIGIN_LENGTH = 64; +const MAX_USER_AGENT_LENGTH = 96; + +/** + * Renders what a request says about its caller — `X-Client`, the requesting site, and the user + * agent — for a log line. + * + * On an endpoint that runs without authentication these headers are all a log line has to go on: + * without them, a partner integration, one of our own apps and a third-party script are the same + * anonymous caller. Of the requesting URL only the origin is used — never its path or query, which + * can carry personal data or tokens — so a browser-side caller can be named by site. + */ +export function describeCaller(req: Request): string { + // The exception filter is the last line of defence: a request object without headers (a + // non-HTTP execution context) must not turn a rejected request into a 500 in here. + if (!req?.headers) return 'client=(none)'; + + const parts = [`client=${maskLogValue(getClient(req), MAX_CLIENT_LENGTH) || '(none)'}`]; + + const origin = callerOrigin(req); + if (origin) parts.push(`origin=${maskLogValue(origin, MAX_ORIGIN_LENGTH)}`); + + const userAgent = firstHeader(req, 'user-agent'); + if (userAgent) parts.push(`ua=${maskLogValue(userAgent, MAX_USER_AGENT_LENGTH)}`); + + return parts.join(' '); +} + +// Both headers are reduced to their origin, `Origin` included: it is supposed to carry nothing +// else, but it arrives from the client like everything here, and a value that is not what it is +// supposed to be is exactly the one that must not reach the log with a query string attached. +// +// The first header that yields an origin wins, not the first one that is present: an `Origin` the +// client filled with something else should cost its own attribution, not the `Referer`'s too. +function callerOrigin(req: Request): string { + for (const header of ['origin', 'referer']) { + const origin = toOrigin(firstHeader(req, header)); + if (origin) return origin; + } + + return ''; +} + +function toOrigin(url: string): string { + // What a browser sends for an opaque origin — a sandboxed frame, a redirect across sites. It is + // not a URL and cannot be reduced to one, but "the caller has no origin to name" is itself worth + // the line, and it is a fixed word rather than anything the client composed. + if (url === 'null') return url; + + try { + return new URL(url).origin; + } catch { + // Not a parsable URL — dropped rather than logged raw, since the unparsed value would be the + // one that is not reduced to its origin. + return ''; + } +} + +// A tampered header can arrive as an array (CodeQL js/type-confusion-through-parameter-tampering). +function firstHeader(req: Request, name: string): string { + const value = req.headers[name]; + return ((Array.isArray(value) ? value[0] : value) ?? '').trim(); +} diff --git a/src/subdomains/core/buy-crypto/routes/buy/dto/__tests__/get-buy-payment-info.dto.spec.ts b/src/subdomains/core/buy-crypto/routes/buy/dto/__tests__/get-buy-payment-info.dto.spec.ts index 7af3cf6b34..eb337f05c3 100644 --- a/src/subdomains/core/buy-crypto/routes/buy/dto/__tests__/get-buy-payment-info.dto.spec.ts +++ b/src/subdomains/core/buy-crypto/routes/buy/dto/__tests__/get-buy-payment-info.dto.spec.ts @@ -1,12 +1,17 @@ -import { ArgumentMetadata, BadRequestException, ValidationPipe } from '@nestjs/common'; +import { ArgumentMetadata, BadRequestException } from '@nestjs/common'; +import { + DetailedValidationPipe, + ValidationFailedException, + describeRejectedValues, +} from 'src/shared/pipes/detailed-validation.pipe'; import { FiatPaymentMethod } from 'src/subdomains/supporting/payment/dto/payment-method.enum'; import { QuoteError } from 'src/subdomains/supporting/payment/dto/transaction-helper/quote-error.enum'; import { GetBuyPaymentInfoDto, PersonalIbanProvider } from '../get-buy-payment-info.dto'; describe('GetBuyPaymentInfoDto.personalIbanProvider', () => { // Mirror the production global pipe (src/main.ts): custom decorator messages must surface as-is - // in the 400 body (no exceptionFactory override). - const pipe = new ValidationPipe({ whitelist: true, transformOptions: { exposeUnsetFields: false } }); + // in the 400 body. + const pipe = new DetailedValidationPipe({ whitelist: true, transformOptions: { exposeUnsetFields: false } }); const metadata: ArgumentMetadata = { type: 'body', metatype: GetBuyPaymentInfoDto, data: '' }; const validBody = { @@ -40,4 +45,33 @@ describe('GetBuyPaymentInfoDto.personalIbanProvider', () => { expect(messageText).toContain(QuoteError.PERSONAL_IBAN_PROVIDER_UNSUPPORTED); expect(messageText).not.toMatch(/must be one of the following values/i); }); + + it('names a provider that differs only in case, despite the field name carrying `iban`', async () => { + // The declared value wins over the name-based redaction: what it renders is this program's + // constant, and the message alone would not say which provider the client meant. + let caught: unknown; + try { + await pipe.transform({ ...validBody, personalIbanProvider: 'frick' }, metadata); + } catch (error) { + caught = error; + } + + expect(caught).toBeInstanceOf(ValidationFailedException); + expect(describeRejectedValues((caught as ValidationFailedException).validationErrors)).toBe( + "personalIbanProvider='Frick'", + ); + }); + + it('still redacts a provider value the field never declared', async () => { + let caught: unknown; + try { + await pipe.transform({ ...validBody, personalIbanProvider: 'CH9300762011623852957' }, metadata); + } catch (error) { + caught = error; + } + + expect(describeRejectedValues((caught as ValidationFailedException).validationErrors)).toBe( + 'personalIbanProvider=***', + ); + }); }); diff --git a/src/subdomains/core/buy-crypto/routes/buy/dto/__tests__/get-buy-quote.dto.spec.ts b/src/subdomains/core/buy-crypto/routes/buy/dto/__tests__/get-buy-quote.dto.spec.ts new file mode 100644 index 0000000000..45c1278e74 --- /dev/null +++ b/src/subdomains/core/buy-crypto/routes/buy/dto/__tests__/get-buy-quote.dto.spec.ts @@ -0,0 +1,51 @@ +import { ArgumentMetadata } from '@nestjs/common'; +import { + DetailedValidationPipe, + ValidationFailedException, + describeRejectedValues, +} from 'src/shared/pipes/detailed-validation.pipe'; +import { GetBuyQuoteDto } from '../get-buy-quote.dto'; + +// The 400 body names the field and, where the field declares them, the accepted values - never the +// value that arrived. These cases pin what the log line shows instead, for the two rejections +// covered below. +describe('GetBuyQuoteDto rejections', () => { + // Mirrors the production global pipe (src/main.ts). + const pipe = new DetailedValidationPipe({ whitelist: true, transformOptions: { exposeUnsetFields: false } }); + const metadata: ArgumentMetadata = { type: 'body', metatype: GetBuyQuoteDto, data: '' }; + + const body = { currency: { id: 1 }, asset: { id: 1 } }; + + async function rejectionDetail(input: Record): Promise { + try { + await pipe.transform(input, metadata); + } catch (error) { + expect(error).toBeInstanceOf(ValidationFailedException); + return describeRejectedValues((error as ValidationFailedException).validationErrors); + } + + throw new Error('expected the body to be rejected'); + } + + it('names the payment method that was sent, not just the accepted ones', async () => { + // 'Crypto' is a value of the wider `PaymentMethod` union (payment-method.enum.ts); this DTO + // accepts `FiatPaymentMethod` only. + await expect(rejectionDetail({ ...body, amount: 100, paymentMethod: 'Crypto' })).resolves.toBe( + "paymentMethod='Crypto'", + ); + }); + + it('distinguishes both amounts missing from both amounts set', async () => { + await expect(rejectionDetail({ ...body })).resolves.toBe('amount=(missing), targetAmount=(missing)'); + await expect(rejectionDetail({ ...body, amount: 100, targetAmount: 1 })).resolves.toBe( + 'amount=, targetAmount=', + ); + }); + + it('accepts a valid body unchanged', async () => { + const dto = await pipe.transform({ ...body, amount: 100, paymentMethod: 'Bank' }, metadata); + + expect(dto.amount).toBe(100); + expect(dto.paymentMethod).toBe('Bank'); + }); +}); diff --git a/src/subdomains/core/buy-crypto/routes/buy/dto/get-buy-payment-info.dto.ts b/src/subdomains/core/buy-crypto/routes/buy/dto/get-buy-payment-info.dto.ts index 7a388d9ca1..8813cd99bf 100644 --- a/src/subdomains/core/buy-crypto/routes/buy/dto/get-buy-payment-info.dto.ts +++ b/src/subdomains/core/buy-crypto/routes/buy/dto/get-buy-payment-info.dto.ts @@ -13,13 +13,14 @@ import { ValidateNested, } from 'class-validator'; import { EntityDto } from 'src/shared/dto/entity.dto'; +import { LogRejectedValue } from 'src/shared/decorators/log-rejected-value.decorator'; import { Asset } from 'src/shared/models/asset/asset.entity'; import { AssetInDto } from 'src/shared/models/asset/dto/asset.dto'; import { Fiat } from 'src/shared/models/fiat/fiat.entity'; import { Util } from 'src/shared/utils/util'; import { XOR } from 'src/shared/validators/xor.validator'; import { IbanType, IsDfxIban } from 'src/subdomains/supporting/bank/bank-account/is-dfx-iban.validator'; -import { FiatPaymentMethod } from 'src/subdomains/supporting/payment/dto/payment-method.enum'; +import { FiatPaymentMethod, PaymentMethodSwagger } from 'src/subdomains/supporting/payment/dto/payment-method.enum'; import { QuoteError } from 'src/subdomains/supporting/payment/dto/transaction-helper/quote-error.enum'; import { PersonalIbanProvider } from './personal-iban-provider.enum'; @@ -61,6 +62,8 @@ export class GetBuyPaymentInfoDto { @IsNotEmpty() @IsEnum(FiatPaymentMethod) + // The crypto members of the union are what a client sends here by mistake. + @LogRejectedValue(PaymentMethodSwagger) paymentMethod: FiatPaymentMethod = FiatPaymentMethod.BANK; @ApiPropertyOptional({ @@ -69,6 +72,8 @@ export class GetBuyPaymentInfoDto { }) @IsOptional() @IsEnum(PersonalIbanProvider, { message: QuoteError.PERSONAL_IBAN_PROVIDER_UNSUPPORTED }) + // The field's own values: a wrong one that differs only in case is named back as the constant. + @LogRejectedValue(PersonalIbanProvider) personalIbanProvider?: PersonalIbanProvider; @ApiPropertyOptional({ description: 'Custom transaction id' }) diff --git a/src/subdomains/core/buy-crypto/routes/buy/dto/get-buy-quote.dto.ts b/src/subdomains/core/buy-crypto/routes/buy/dto/get-buy-quote.dto.ts index 38017e0500..572137dbdb 100644 --- a/src/subdomains/core/buy-crypto/routes/buy/dto/get-buy-quote.dto.ts +++ b/src/subdomains/core/buy-crypto/routes/buy/dto/get-buy-quote.dto.ts @@ -11,12 +11,13 @@ import { ValidateIf, ValidateNested, } from 'class-validator'; +import { LogRejectedValue } from 'src/shared/decorators/log-rejected-value.decorator'; import { Asset } from 'src/shared/models/asset/asset.entity'; import { AssetInDto } from 'src/shared/models/asset/dto/asset.dto'; import { Fiat } from 'src/shared/models/fiat/fiat.entity'; import { FiatInDto } from 'src/shared/models/fiat/dto/fiat.dto'; import { XOR } from 'src/shared/validators/xor.validator'; -import { FiatPaymentMethod } from 'src/subdomains/supporting/payment/dto/payment-method.enum'; +import { FiatPaymentMethod, PaymentMethodSwagger } from 'src/subdomains/supporting/payment/dto/payment-method.enum'; export class GetBuyQuoteDto { @ApiProperty({ type: FiatInDto, description: 'Source currency (by ID or name)' }) @@ -48,6 +49,8 @@ export class GetBuyQuoteDto { @ApiPropertyOptional({ description: 'Payment method', enum: FiatPaymentMethod }) @IsNotEmpty() @IsEnum(FiatPaymentMethod) + // The crypto members of the union are what a client sends here by mistake. + @LogRejectedValue(PaymentMethodSwagger) paymentMethod: FiatPaymentMethod = FiatPaymentMethod.BANK; @ApiPropertyOptional({ description: 'This field is deprecated, use "specialCode" instead.', deprecated: true }) From dc50ea30dc531cba16c1480abc55bbb932da8ff4 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Sat, 1 Aug 2026 11:29:04 +0200 Subject: [PATCH 4/4] feat(auth): tell staff why an elevated endpoint is refused (#4566) A staff member who holds the role but is not KYC-cleared got the generic "Forbidden resource", which is indistinguishable from a removed role. Neither the person nor any tooling in front of the API could tell that the fix is to complete an identification. RoleGuard now throws StaffKycRequiredException: HTTP 403 with a machine-readable code STAFF_KYC_REQUIRED and a message naming the actual requirement. This follows the existing TFA_REQUIRED pattern, so clients branch on the code instead of matching prose. A wrong ROLE still yields the generic 403 - the two cases need different actions and must stay distinguishable. The protected KYC file route checked both conditions through one predicate and reported "Requires admin or compliance role" even when the role was fine; it now reports each case separately. JwtUserActiveGuard calls the guard programmatically, but only with UserRole.USER, which is never elevated - so it still receives a boolean. --- jest.staff-gate.config.js | 6 +++ package.json | 2 +- src/shared/auth/__tests__/role.guard.spec.ts | 45 ++++++++++++++++--- .../staff-kyc-required.exception.ts | 15 +++++++ src/shared/auth/role.guard.ts | 8 +++- .../generic/kyc/services/kyc.service.ts | 16 ++++--- 6 files changed, 79 insertions(+), 13 deletions(-) create mode 100644 src/shared/auth/exceptions/staff-kyc-required.exception.ts diff --git a/jest.staff-gate.config.js b/jest.staff-gate.config.js index 5c117b1a0b..ae5299d9e2 100644 --- a/jest.staff-gate.config.js +++ b/jest.staff-gate.config.js @@ -16,6 +16,12 @@ module.exports = { coverageThreshold: { 'src/shared/auth/role.guard.ts': { branches: 100, functions: 100, lines: 100, statements: 100 }, 'src/shared/auth/staff-kyc-clearance.ts': { branches: 100, functions: 100, lines: 100, statements: 100 }, + 'src/shared/auth/exceptions/staff-kyc-required.exception.ts': { + branches: 100, + functions: 100, + lines: 100, + statements: 100, + }, 'src/subdomains/generic/user/models/user/staff-kyc-clearance.service.ts': { branches: 100, functions: 100, diff --git a/package.json b/package.json index 8161508d2d..c47a2bed43 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "test:cov": "jest --coverage", "test:frick:cov": "jest --config jest.frick.config.js integration/bank/services/__tests__/frick.service.spec.ts integration/bank/services/__tests__/iso20022.service.spec.ts config/__tests__/frick.config.spec.ts config/__tests__/bank-frick-config.spec.ts subdomains/supporting/bank-tx/bank-tx/services/__tests__/bank-tx-frick.service.spec.ts subdomains/supporting/bank-tx/bank-tx/services/__tests__/bank-tx-outgoing-match.service.spec.ts subdomains/supporting/fiat-output/__tests__/fiat-output-frick.service.spec.ts subdomains/supporting/bank/virtual-iban/__tests__/virtual-iban-frick-issuance-reconciliation.service.spec.ts subdomains/supporting/bank/virtual-iban/__tests__/virtual-iban.service.spec.ts subdomains/supporting/bank/virtual-iban/providers/__tests__/frick-viban.provider.spec.ts --coverage --runInBand --collectCoverageFrom=integration/bank/dto/frick.dto.ts --collectCoverageFrom=integration/bank/services/frick.service.ts --collectCoverageFrom=integration/bank/services/iso20022.service.ts --collectCoverageFrom=config/frick.config.ts --collectCoverageFrom=subdomains/supporting/bank-tx/bank-tx/services/bank-tx-frick.service.ts --collectCoverageFrom=subdomains/supporting/bank-tx/bank-tx/services/bank-tx-outgoing-match.service.ts --collectCoverageFrom=subdomains/supporting/fiat-output/fiat-output-frick.service.ts --collectCoverageFrom=subdomains/supporting/bank/virtual-iban/virtual-iban-frick-issuance-reconciliation.service.ts --collectCoverageFrom=subdomains/supporting/bank/virtual-iban/virtual-iban.service.ts --collectCoverageFrom=subdomains/supporting/bank/virtual-iban/providers/frick-viban.provider.ts", "test:gate:cov": "jest --config jest.coverage-gate.config.js --coverage --silent", - "test:staff-gate:cov": "jest --config jest.staff-gate.config.js shared/auth/__tests__/role.guard.spec.ts shared/auth/__tests__/staff-kyc-clearance.spec.ts subdomains/generic/user/models/user/__tests__/staff-kyc-clearance.service.spec.ts --coverage --runInBand --collectCoverageFrom=shared/auth/role.guard.ts --collectCoverageFrom=shared/auth/staff-kyc-clearance.ts --collectCoverageFrom=subdomains/generic/user/models/user/staff-kyc-clearance.service.ts", + "test:staff-gate:cov": "jest --config jest.staff-gate.config.js shared/auth/__tests__/role.guard.spec.ts shared/auth/__tests__/staff-kyc-clearance.spec.ts subdomains/generic/user/models/user/__tests__/staff-kyc-clearance.service.spec.ts --coverage --runInBand --collectCoverageFrom=shared/auth/role.guard.ts --collectCoverageFrom=shared/auth/staff-kyc-clearance.ts --collectCoverageFrom=shared/auth/exceptions/staff-kyc-required.exception.ts --collectCoverageFrom=subdomains/generic/user/models/user/staff-kyc-clearance.service.ts", "type-check": "tsc --noEmit", "format:check": "prettier --check \"src/**/*.ts\" \"test/**/*.ts\"", "check": "npm run lint && npm run test", diff --git a/src/shared/auth/__tests__/role.guard.spec.ts b/src/shared/auth/__tests__/role.guard.spec.ts index 94c5994ff3..5f4637084b 100644 --- a/src/shared/auth/__tests__/role.guard.spec.ts +++ b/src/shared/auth/__tests__/role.guard.spec.ts @@ -1,4 +1,4 @@ -import { ExecutionContext } from '@nestjs/common'; +import { ExecutionContext, HttpStatus } from '@nestjs/common'; // The clearance Set is primed by cron; mocked here so the guard's gating logic is tested in isolation // from the cron/DB plumbing. @@ -6,6 +6,7 @@ jest.mock('src/shared/auth/staff-kyc-clearance', () => ({ HasStaffKycClearance: jest.fn(), })); +import { StaffKycRequiredException } from 'src/shared/auth/exceptions/staff-kyc-required.exception'; import { HasStaffKycClearance } from 'src/shared/auth/staff-kyc-clearance'; import { hasRoleAccess, hasStaffAccess, RoleGuard, rolesSatisfying } from '../role.guard'; import { KycGatedRoles, UserRole } from '../user-role.enum'; @@ -198,7 +199,8 @@ describe('RoleGuard (staff KYC gate on elevated endpoints)', () => { it('denies the matching role when the account has no KYC clearance', () => { hasStaffKycClearanceMock.mockReturnValue(false); - expect(RoleGuard(entryRole).canActivate(contextFor(entryRole, 42))).toBe(false); + // Throws rather than returning false, so the caller learns the reason instead of a bare 403. + expect(() => RoleGuard(entryRole).canActivate(contextFor(entryRole, 42))).toThrow(StaffKycRequiredException); }); it('grants the matching role when the account is cleared', () => { @@ -212,10 +214,12 @@ describe('RoleGuard (staff KYC gate on elevated endpoints)', () => { it('gates super-roles too — an uncleared ADMIN loses every elevated endpoint', () => { hasStaffKycClearanceMock.mockReturnValue(false); - expect(RoleGuard(UserRole.SUPPORT).canActivate(contextFor(UserRole.ADMIN, 42))).toBe(false); - expect(RoleGuard(UserRole.COMPLIANCE, UserRole.DEBUG).canActivate(contextFor(UserRole.SUPER_ADMIN, 42))).toBe( - false, + expect(() => RoleGuard(UserRole.SUPPORT).canActivate(contextFor(UserRole.ADMIN, 42))).toThrow( + StaffKycRequiredException, ); + expect(() => + RoleGuard(UserRole.COMPLIANCE, UserRole.DEBUG).canActivate(contextFor(UserRole.SUPER_ADMIN, 42)), + ).toThrow(StaffKycRequiredException); }); it('does not gate ordinary endpoints, even for a staff caller without clearance', () => { @@ -240,6 +244,35 @@ describe('RoleGuard (staff KYC gate on elevated endpoints)', () => { // letting `undefined` slip through the clearance lookup. hasStaffKycClearanceMock.mockImplementation((account) => account != null); - expect(RoleGuard(UserRole.ADMIN).canActivate(contextFor(UserRole.ADMIN, undefined))).toBe(false); + expect(() => RoleGuard(UserRole.ADMIN).canActivate(contextFor(UserRole.ADMIN, undefined))).toThrow( + StaffKycRequiredException, + ); + }); + + // The point of throwing: a client — a person, a script, or an agent — must be able to tell this apart + // from a removed role without matching on prose. + it('answers 403 with a machine-readable code and an actionable message', () => { + hasStaffKycClearanceMock.mockReturnValue(false); + + let thrown: StaffKycRequiredException; + try { + RoleGuard(UserRole.ADMIN).canActivate(contextFor(UserRole.ADMIN, 42)); + } catch (e) { + thrown = e as StaffKycRequiredException; + } + + expect(thrown.getStatus()).toBe(HttpStatus.FORBIDDEN); + expect(thrown.getResponse()).toEqual({ + code: 'STAFF_KYC_REQUIRED', + message: expect.stringContaining('KYC level 50'), + }); + }); + + // A wrong role is a different situation with a different fix, so it must not produce the KYC answer. + it('still returns a plain false when the role itself does not satisfy the gate', () => { + hasStaffKycClearanceMock.mockReturnValue(false); + + expect(RoleGuard(UserRole.ADMIN).canActivate(contextFor(UserRole.USER, 42))).toBe(false); + expect(hasStaffKycClearanceMock).not.toHaveBeenCalled(); }); }); diff --git a/src/shared/auth/exceptions/staff-kyc-required.exception.ts b/src/shared/auth/exceptions/staff-kyc-required.exception.ts new file mode 100644 index 0000000000..c93d96ae5a --- /dev/null +++ b/src/shared/auth/exceptions/staff-kyc-required.exception.ts @@ -0,0 +1,15 @@ +import { ForbiddenException } from '@nestjs/common'; + +// Answer for a staff caller who holds the role but is not KYC-cleared (see `HasStaffKycClearance`). +// A bare `false` from the guard would produce the generic "Forbidden resource", which is +// indistinguishable from "your role was removed" — leaving the caller, and any tooling in front of +// it, with no way to tell that the fix is to complete an identification. The machine-readable `code` +// follows the existing `TFA_REQUIRED` pattern so clients can branch on it instead of matching text. +export class StaffKycRequiredException extends ForbiddenException { + constructor() { + super({ + code: 'STAFF_KYC_REQUIRED', + message: 'Staff access requires a completed identification: KYC level 50 and a verified name on your account', + }); + } +} diff --git a/src/shared/auth/role.guard.ts b/src/shared/auth/role.guard.ts index 308d25d452..cd8dfe0ce4 100644 --- a/src/shared/auth/role.guard.ts +++ b/src/shared/auth/role.guard.ts @@ -1,4 +1,5 @@ import { CanActivate, ExecutionContext } from '@nestjs/common'; +import { StaffKycRequiredException } from 'src/shared/auth/exceptions/staff-kyc-required.exception'; import { HasStaffKycClearance } from 'src/shared/auth/staff-kyc-clearance'; import { KycGatedRoles, UserRole } from 'src/shared/auth/user-role.enum'; @@ -94,7 +95,12 @@ class RoleGuardClass implements CanActivate { // The gate is a property of the ENDPOINT, not of the caller, so it applies only when EVERY entry // role is gated — a gate that also admits e.g. UserRole.USER is an ordinary endpoint that an admin // happens to reach through the role hierarchy, and must not start demanding staff KYC. - if (this.isElevated) return HasStaffKycClearance(user?.account); + // + // Throws rather than returning false: a bare false becomes the generic "Forbidden resource", which + // a caller cannot tell apart from a removed role, so neither staff nor tooling would learn that the + // fix is to complete an identification. `JwtUserActiveGuard` calls this guard programmatically, but + // only with UserRole.USER, which is never elevated — so it still gets a boolean. + if (this.isElevated && !HasStaffKycClearance(user?.account)) throw new StaffKycRequiredException(); return true; } diff --git a/src/subdomains/generic/kyc/services/kyc.service.ts b/src/subdomains/generic/kyc/services/kyc.service.ts index dc68c325bc..92de4fa10f 100644 --- a/src/subdomains/generic/kyc/services/kyc.service.ts +++ b/src/subdomains/generic/kyc/services/kyc.service.ts @@ -9,7 +9,9 @@ import { import { CronExpression } from '@nestjs/schedule'; import { Config } from 'src/config/config'; import { JwtPayload } from 'src/shared/auth/jwt-payload.interface'; -import { hasStaffAccess } from 'src/shared/auth/role.guard'; +import { StaffKycRequiredException } from 'src/shared/auth/exceptions/staff-kyc-required.exception'; +import { hasRoleAccess } from 'src/shared/auth/role.guard'; +import { HasStaffKycClearance } from 'src/shared/auth/staff-kyc-clearance'; import { isUserActive } from 'src/shared/auth/user-active.guard'; import { UserRole } from 'src/shared/auth/user-role.enum'; import { Country } from 'src/shared/models/country/country.entity'; @@ -473,10 +475,14 @@ export class KycService { if (!kycFile) throw new NotFoundException('KYC file not found'); if (kycFile.protected) { - // `hasStaffAccess`, not `hasRoleAccess`: this route is OptionalJwtAuthGuard-only, so no RoleGuard has - // applied the staff KYC gate. Protected KYC files are the most sensitive sink in the API — an - // uncleared compliance/admin account must not reach them just because the endpoint is not role-gated. - if (!hasStaffAccess(UserRole.COMPLIANCE, jwt)) throw new ForbiddenException('Requires admin or compliance role'); + // This route is OptionalJwtAuthGuard-only, so no RoleGuard has applied the staff KYC gate. Protected + // KYC files are the most sensitive sink in the API — an uncleared compliance/admin account must not + // reach them just because the endpoint is not role-gated. The two conditions are checked separately + // so the caller learns which one failed: "wrong role" and "role fine, identification missing" need + // different actions, and a single message for both sends staff looking in the wrong place. + if (!hasRoleAccess(UserRole.COMPLIANCE, jwt?.role)) + throw new ForbiddenException('Requires admin or compliance role'); + if (!HasStaffKycClearance(jwt?.account)) throw new StaffKycRequiredException(); if (!jwt || !isUserActive(jwt)) throw new ForbiddenException('User is not active'); // Mail-origin staff sessions (tfaRequired) must complete STRICT 2FA before downloading protected KYC