diff --git a/.github/workflows/api-pr.yaml b/.github/workflows/api-pr.yaml index 52675a7cba..2e1b12ec70 100644 --- a/.github/workflows/api-pr.yaml +++ b/.github/workflows/api-pr.yaml @@ -83,6 +83,11 @@ jobs: - name: Run coverage run: npm run test:frick:cov + # Same mechanics for the staff KYC gate: the files deciding who reaches an elevated endpoint are + # pinned at 100%, so an uncovered branch in the authorization path fails the PR. + - name: Run staff gate coverage + run: npm run test:staff-gate:cov + # Runs on a self-hosted runner for branches of this repository. The gate executes the whole suite # under full compilation and is CPU-bound; a hosted runner gives a public repo four vCPUs, so Jest # defaults to three workers and the gate alone decided how long a PR run took. The self-hosted diff --git a/jest.coverage-gate.config.js b/jest.coverage-gate.config.js index 3374a71660..093831ff80 100644 --- a/jest.coverage-gate.config.js +++ b/jest.coverage-gate.config.js @@ -62,12 +62,14 @@ const PINNED_LOGIC = [ 'src/shared/auth/allow-tfa-pending.decorator.ts', 'src/shared/auth/get-jwt.decorator.ts', 'src/shared/auth/user-role.enum.ts', + 'src/shared/decorators/log-rejected-value.decorator.ts', 'src/shared/services/typeorm-logger.ts', 'src/shared/utils/bitbox-ascii.util.ts', 'src/shared/utils/cron.ts', 'src/shared/utils/custom-cron-expression.ts', 'src/shared/utils/request-client.ts', 'src/shared/validators/is-ssrf-safe-url.validator.ts', + 'src/shared/validators/xor.validator.ts', 'src/subdomains/core/accounting/controllers/ledger.controller.ts', 'src/subdomains/core/accounting/dto/ledger-account.dto.ts', 'src/subdomains/core/accounting/dto/ledger-dto.mapper.ts', @@ -92,6 +94,7 @@ const PINNED_LOGIC = [ 'src/subdomains/core/aml/enums/scorechain-outcome.enum.ts', 'src/subdomains/core/aml/services/transaction-aml-check.service.ts', 'src/subdomains/core/buy-crypto/process/exceptions/abort-batch-creation.exception.ts', + 'src/subdomains/core/buy-crypto/routes/buy/dto/get-buy-quote.dto.ts', 'src/subdomains/core/buy-crypto/routes/buy/dto/personal-iban-provider.enum.ts', 'src/subdomains/core/custody/dto/output/custody-order-history.dto.ts', 'src/subdomains/core/custody/enums/custody.ts', diff --git a/jest.staff-gate.config.js b/jest.staff-gate.config.js new file mode 100644 index 0000000000..ae5299d9e2 --- /dev/null +++ b/jest.staff-gate.config.js @@ -0,0 +1,32 @@ +// Staff KYC gate coverage. Kept out of package.json's shared Jest config (same reasoning as +// jest.frick.config.js) so the strict per-file 100% threshold cannot red an unrelated `test:cov` run — +// only the dedicated test:staff-gate:cov step, with its own --collectCoverageFrom scope, enforces it. +// +// These three files decide who reaches every elevated endpoint. A partially covered branch here is an +// unreviewed hole in the authorization path, so they are pinned at 100% on all four metrics. +const base = require('./package.json').jest; + +module.exports = { + ...base, + // Coverage instrumentation must match the production build's emit. The main suite runs ts-jest in + // transpile-only mode (isolatedModules), which emits the emitDecoratorMetadata helpers differently + // and produces phantom uncovered branches on dependency-injected constructors. Compile with full + // type info here (tsconfig.coverage.json sets isolatedModules: false) so the 100% gate stays exact. + transform: { '^.+\\.(t|j)s$': ['ts-jest', { tsconfig: 'tsconfig.coverage.json' }] }, + coverageThreshold: { + 'src/shared/auth/role.guard.ts': { branches: 100, functions: 100, lines: 100, statements: 100 }, + 'src/shared/auth/staff-kyc-clearance.ts': { branches: 100, functions: 100, lines: 100, statements: 100 }, + 'src/shared/auth/exceptions/staff-kyc-required.exception.ts': { + branches: 100, + functions: 100, + lines: 100, + statements: 100, + }, + 'src/subdomains/generic/user/models/user/staff-kyc-clearance.service.ts': { + branches: 100, + functions: 100, + lines: 100, + statements: 100, + }, + }, +}; diff --git a/package.json b/package.json index 74a9f61938..c47a2bed43 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "test:cov": "jest --coverage", "test:frick:cov": "jest --config jest.frick.config.js integration/bank/services/__tests__/frick.service.spec.ts integration/bank/services/__tests__/iso20022.service.spec.ts config/__tests__/frick.config.spec.ts config/__tests__/bank-frick-config.spec.ts subdomains/supporting/bank-tx/bank-tx/services/__tests__/bank-tx-frick.service.spec.ts subdomains/supporting/bank-tx/bank-tx/services/__tests__/bank-tx-outgoing-match.service.spec.ts subdomains/supporting/fiat-output/__tests__/fiat-output-frick.service.spec.ts subdomains/supporting/bank/virtual-iban/__tests__/virtual-iban-frick-issuance-reconciliation.service.spec.ts subdomains/supporting/bank/virtual-iban/__tests__/virtual-iban.service.spec.ts subdomains/supporting/bank/virtual-iban/providers/__tests__/frick-viban.provider.spec.ts --coverage --runInBand --collectCoverageFrom=integration/bank/dto/frick.dto.ts --collectCoverageFrom=integration/bank/services/frick.service.ts --collectCoverageFrom=integration/bank/services/iso20022.service.ts --collectCoverageFrom=config/frick.config.ts --collectCoverageFrom=subdomains/supporting/bank-tx/bank-tx/services/bank-tx-frick.service.ts --collectCoverageFrom=subdomains/supporting/bank-tx/bank-tx/services/bank-tx-outgoing-match.service.ts --collectCoverageFrom=subdomains/supporting/fiat-output/fiat-output-frick.service.ts --collectCoverageFrom=subdomains/supporting/bank/virtual-iban/virtual-iban-frick-issuance-reconciliation.service.ts --collectCoverageFrom=subdomains/supporting/bank/virtual-iban/virtual-iban.service.ts --collectCoverageFrom=subdomains/supporting/bank/virtual-iban/providers/frick-viban.provider.ts", "test:gate:cov": "jest --config jest.coverage-gate.config.js --coverage --silent", + "test:staff-gate:cov": "jest --config jest.staff-gate.config.js shared/auth/__tests__/role.guard.spec.ts shared/auth/__tests__/staff-kyc-clearance.spec.ts subdomains/generic/user/models/user/__tests__/staff-kyc-clearance.service.spec.ts --coverage --runInBand --collectCoverageFrom=shared/auth/role.guard.ts --collectCoverageFrom=shared/auth/staff-kyc-clearance.ts --collectCoverageFrom=shared/auth/exceptions/staff-kyc-required.exception.ts --collectCoverageFrom=subdomains/generic/user/models/user/staff-kyc-clearance.service.ts", "type-check": "tsc --noEmit", "format:check": "prettier --check \"src/**/*.ts\" \"test/**/*.ts\"", "check": "npm run lint && npm run test", diff --git a/src/integration/exchange/dto/scrypt.dto.ts b/src/integration/exchange/dto/scrypt.dto.ts index 4905bd82a8..01eccd7e00 100644 --- a/src/integration/exchange/dto/scrypt.dto.ts +++ b/src/integration/exchange/dto/scrypt.dto.ts @@ -107,11 +107,51 @@ export enum ScryptOrderStatus { PENDING_REPLACE = 'PendingReplace', } +/** + * Terminal order statuses at Scrypt: nothing under the reference can still execute. + * + * Single source of truth for both execution-report caching (`OrdStatus`) and order-status lookups + * (`ScryptOrderInfo.status`). Do not re-list these three values elsewhere. + */ +export const SCRYPT_TERMINAL_ORDER_STATUSES: readonly ScryptOrderStatus[] = [ + ScryptOrderStatus.FILLED, + ScryptOrderStatus.CANCELED, + ScryptOrderStatus.REJECTED, +]; + +export function isTerminalScryptOrderStatus(status: ScryptOrderStatus): boolean { + return (SCRYPT_TERMINAL_ORDER_STATUSES as readonly ScryptOrderStatus[]).includes(status); +} + export enum ScryptOrderSide { BUY = 'Buy', SELL = 'Sell', } +/** + * What asking the venue to cancel a reference established about it. + * + * Three outcomes, not two, because "cancelled" does not mean "nothing happened": a partially filled order + * is cancelled with a terminal status AND a fill, and that fill is worth naming rather than folding into + * the same answer as an untouched one. + */ +export enum ScryptCancellation { + /** + * Nothing can execute under this reference any more. Two different qualities of answer: cancelled with + * nothing filled settles it outright, while the venue not knowing the reference is an inference from its + * own words — see SCRYPT_UNKNOWN_ORDER for what that evidence covers. + */ + SETTLED = 'Settled', + /** + * It reached a terminal state with something filled. Like a cancelled reference it cannot trade further, + * so the order may be given up — the fill already moved the venue balance the rule replans from. Kept + * distinct from SETTLED because a fill is worth seeing in a log and worth reconciling against. + */ + EXECUTED = 'Executed', + /** No usable answer. Nothing may be concluded, least of all that the reference is safe to walk away from. */ + UNCONFIRMED = 'Unconfirmed', +} + export enum ScryptOrderType { MARKET = 'Market', LIMIT = 'Limit', diff --git a/src/integration/exchange/services/__tests__/scrypt.service.spec.ts b/src/integration/exchange/services/__tests__/scrypt.service.spec.ts index 1a24960be8..1afa09e800 100644 --- a/src/integration/exchange/services/__tests__/scrypt.service.spec.ts +++ b/src/integration/exchange/services/__tests__/scrypt.service.spec.ts @@ -2,6 +2,8 @@ import { GetConfig } from 'src/config/config'; import { Util } from 'src/shared/utils/util'; import { ScryptBalanceTransaction, + ScryptCancellation, + ScryptExecutionReport, ScryptOrderStatus, ScryptTransactionStatus, ScryptTransactionType, @@ -9,6 +11,7 @@ import { import { ScryptAmendRejectedError, ScryptMessageType, + ScryptOrderStuckPendingError, ScryptRequestTimeoutError, ScryptUnconfirmedWriteError, ScryptVenueRejectionError, @@ -361,15 +364,20 @@ describe('ScryptService', () => { expect((service as any).lastCatchUpAt).toBeGreaterThan(startStamp!); }); - it('a warm-up that failed does not claim the catch-up slot', async () => { + it('a warm-up that failed schedules catch-up without waiting for a reconnect', async () => { + // After FIX 2 a failed warm-up immediately reuses catchUpAfterReconnect. That stamps lastCatchUpAt at + // the end of every round (pacing, independent of success) and arms catchUpRetryTimer when legs stay + // owed — so lastCatchUpAt is no longer undefined after a failed warm-up. The invariant that matters is + // that a retry is scheduled without onReconnect ever firing the registered callback. const MockedConnection = ScryptWebSocketConnection as jest.MockedClass; + const onReconnect = jest.fn(); MockedConnection.mockImplementationOnce( () => ({ fetchAll: jest.fn().mockRejectedValue(new Error('Connection closed')), fetch: jest.fn().mockResolvedValue([]), subscribeToStream: jest.fn().mockReturnValue(() => undefined), - onReconnect: jest.fn(), + onReconnect, send: jest.fn(), requestAndWaitForUpdate: jest.fn(), }) as any, @@ -377,10 +385,57 @@ describe('ScryptService', () => { const freshService = new ScryptService(); jest.spyOn((freshService as any).logger, 'error').mockImplementation(() => undefined); + jest.spyOn((freshService as any).logger, 'warn').mockImplementation(() => undefined); await flushPromises(); - // The caches are not whole, so the next reconnect must repair immediately instead of sitting out the interval. - expect((freshService as any).lastCatchUpAt).toBeUndefined(); + // onReconnect only registers the handler; we never invoke that callback. The timer proves the + // boot-failure path armed a retry on its own without a reconnect. + expect(onReconnect).toHaveBeenCalled(); + expect(typeof onReconnect.mock.calls[0][0]).toBe('function'); + expect((freshService as any).catchUpRetryTimer).toBeDefined(); + (freshService as any).clearCatchUpRetry(); + }); + + it('a failed boot warm-up retries catch-up without a reconnect and respects catchUpMinInterval', async () => { + // FIX 2: when warm-up rejects and the socket stays up, catchUpAfterReconnect must run immediately and + // its scheduled retry must actually re-enter after catchUpMinInterval — otherwise the empty cache is a + // permanent deferral (anchor check forever false) rather than a temporary one. + jest.useFakeTimers(); + const MockedConnection = ScryptWebSocketConnection as jest.MockedClass; + const fetchAll = jest.fn().mockRejectedValue(new Error('Connection closed')); + const onReconnect = jest.fn(); + MockedConnection.mockImplementationOnce( + () => + ({ + fetchAll, + fetch: jest.fn().mockResolvedValue([]), + subscribeToStream: jest.fn().mockReturnValue(() => undefined), + onReconnect, + send: jest.fn(), + requestAndWaitForUpdate: jest.fn(), + }) as any, + ); + + const freshService = new ScryptService(); + jest.spyOn((freshService as any).logger, 'error').mockImplementation(() => undefined); + jest.spyOn((freshService as any).logger, 'warn').mockImplementation(() => undefined); + await flushPromises(); + + // (1) Retry armed without ever invoking the reconnect callback registered via onReconnect. + expect((freshService as any).catchUpRetryTimer).toBeDefined(); + const reconnectHandler = onReconnect.mock.calls[0]?.[0] as (() => void) | undefined; + // Handler is registered; we never call it — refill must not depend on a drop. + expect(typeof reconnectHandler).toBe('function'); + + // (2) The armed retry is not cosmetic: after catchUpMinInterval further fetchAll calls land. + const callsBeforeRetry = fetchAll.mock.calls.length; + expect(callsBeforeRetry).toBeGreaterThan(0); + jest.advanceTimersByTime((freshService as any).catchUpMinInterval); + await flushPromises(); + expect(fetchAll.mock.calls.length).toBeGreaterThan(callsBeforeRetry); + + (freshService as any).clearCatchUpRetry(); + jest.useRealTimers(); }); it('a warm-up that loaded both streams claims the catch-up slot', async () => { @@ -677,6 +732,72 @@ describe('ScryptService', () => { expect((freshService as any).balanceTransactions.get('warm-old')).toBeUndefined(); }); + it('bulk applyBalanceTransactions caches rows without Timestamp when TransactTime is set', async () => { + // Field priority: Timestamp missing → TransactTime; recent stamp must still land in the cache. + const recent = new Date().toISOString(); + const noTimestamp = { + ClReqID: 'warm-transact-time', + TransactionID: 'tx-warm-tt', + Status: ScryptTransactionStatus.COMPLETED, + TransactTime: recent, + }; + + const MockedConnection = ScryptWebSocketConnection as jest.MockedClass; + MockedConnection.mockImplementationOnce( + () => + ({ + fetchAll: jest.fn().mockImplementation(async (streamName: string) => { + if (streamName === ScryptMessageType.BALANCE_TRANSACTION) { + return [noTimestamp]; + } + return []; + }), + fetch: jest.fn().mockResolvedValue([]), + subscribeToStream: jest.fn().mockReturnValue(() => undefined), + onReconnect: jest.fn(), + send: jest.fn(), + requestAndWaitForUpdate: jest.fn(), + }) as any, + ); + + const freshService = new ScryptService(); + await flushPromises(); + + expect((freshService as any).balanceTransactions.get('warm-transact-time')).toEqual(noTimestamp); + }); + + it('bulk applyBalanceTransactions caches rows with neither Timestamp nor TransactTime (conservative)', async () => { + // Missing/unreadable stamp must not drop a withdrawal we later need for findWithdrawal / live recheck. + const noStamp = { + ClReqID: 'warm-no-stamp', + TransactionID: 'tx-warm-no-stamp', + Status: ScryptTransactionStatus.COMPLETED, + }; + + const MockedConnection = ScryptWebSocketConnection as jest.MockedClass; + MockedConnection.mockImplementationOnce( + () => + ({ + fetchAll: jest.fn().mockImplementation(async (streamName: string) => { + if (streamName === ScryptMessageType.BALANCE_TRANSACTION) { + return [noStamp]; + } + return []; + }), + fetch: jest.fn().mockResolvedValue([]), + subscribeToStream: jest.fn().mockReturnValue(() => undefined), + onReconnect: jest.fn(), + send: jest.fn(), + requestAndWaitForUpdate: jest.fn(), + }) as any, + ); + + const freshService = new ScryptService(); + await flushPromises(); + + expect((freshService as any).balanceTransactions.get('warm-no-stamp')).toEqual(noStamp); + }); + it('catchUpAfterReconnect coalesces a reconnect seen during the fetches into one follow-up round', async () => { let fetchAllCallCount = 0; @@ -806,6 +927,399 @@ describe('ScryptService', () => { ]); }); }); + describe('confirmWithdrawalAbsent', () => { + const soughtId = 'dfx-lm-withdraw-9'; + + function balanceTx(overrides: Partial = {}): ScryptBalanceTransaction { + return { + TransactionID: 'tx-1', + ClReqID: 'other-ref', + Currency: 'CHF', + TransactionType: ScryptTransactionType.WITHDRAWAL, + Status: ScryptTransactionStatus.COMPLETED, + Quantity: '100', + Timestamp: '2026-07-15T12:00:00.000Z', + ...overrides, + }; + } + + it('returns true when the cache is empty and the sought reference is absent from a non-empty fresh history', async () => { + // Empty process cache is no longer a reason to refuse: absence is decided from the fresh venue reply + // (plus the live recheck). No local anchor is required. + instance.fetchAll.mockResolvedValue([balanceTx({ ClReqID: 'unrelated-in-history', TransactionID: 'tx-u' })]); + + await expect(service.confirmWithdrawalAbsent(soughtId)).resolves.toBe(true); + }); + + it('returns true when the cache holds only post-since rows and the sought reference is absent from fresh', async () => { + // Cache rows newer than the order no longer act as consistency anchors; missing sought id → true. + (service as any).balanceTransactions.set('recent-only', { + ...balanceTx({ ClReqID: 'recent-only', Timestamp: '2026-07-20T00:00:00.000Z' }), + }); + instance.fetchAll.mockResolvedValue([ + balanceTx({ ClReqID: 'recent-only', TransactionID: 'tx-r', Timestamp: '2026-07-20T00:00:00.000Z' }), + balanceTx({ ClReqID: 'unrelated-in-history', TransactionID: 'tx-u', Timestamp: '2026-07-21T00:00:00.000Z' }), + ]); + + await expect(service.confirmWithdrawalAbsent(soughtId)).resolves.toBe(true); + }); + + it('returns false when the fresh history contains the sought reference', async () => { + instance.fetchAll.mockResolvedValue([balanceTx({ ClReqID: soughtId, TransactionID: 'tx-sought' })]); + + await expect(service.confirmWithdrawalAbsent(soughtId)).resolves.toBe(false); + }); + + it('returns false and warns when the history fetch fails', async () => { + const warnSpy = jest.spyOn(service['logger'], 'warn').mockImplementation(); + instance.fetchAll.mockRejectedValue(new Error('Connection closed')); + + await expect(service.confirmWithdrawalAbsent(soughtId)).resolves.toBe(false); + expect(warnSpy).toHaveBeenCalled(); + }); + + it('returns true when the venue returns an empty history — no rows at all is a complete answer, not a reason to wait', async () => { + const infoSpy = jest.spyOn(service['logger'], 'info').mockImplementation(); + instance.fetchAll.mockResolvedValue([]); + + await expect(service.confirmWithdrawalAbsent(soughtId)).resolves.toBe(true); + expect(infoSpy).toHaveBeenCalled(); + }); + + it('returns false when the sought reference appears in the live cache while the history fetch is in flight', async () => { + // Live push mid-flight lands in this.balanceTransactions; freshIds does not include soughtId. Only the + // post-await re-read of the live map can block absence confirmation for that race. + const warnSpy = jest.spyOn(service['logger'], 'warn').mockImplementation(); + + instance.fetchAll.mockImplementationOnce(async () => { + (service as any).cacheBalanceTransaction( + balanceTx({ ClReqID: soughtId, TransactionID: 'tx-live', Timestamp: '2026-07-10T00:00:00.000Z' }), + ); + return [balanceTx({ ClReqID: 'unrelated', TransactionID: 'tx-u', Timestamp: '2026-07-20T00:00:00.000Z' })]; + }); + + await expect(service.confirmWithdrawalAbsent(soughtId)).resolves.toBe(false); + expect(warnSpy).toHaveBeenCalled(); + expect(warnSpy.mock.calls.some((c) => String(c[0]).includes(soughtId))).toBe(true); + }); + + it('returns true when the sought reference is absent from a non-empty fresh history', async () => { + // Cache content is irrelevant for absence confirmation as long as the live recheck does not hit. + instance.fetchAll.mockResolvedValue([ + balanceTx({ ClReqID: 'unrelated-in-history', TransactionID: 'tx-u', Timestamp: '2026-07-21T00:00:00.000Z' }), + ]); + + await expect(service.confirmWithdrawalAbsent(soughtId)).resolves.toBe(true); + }); + }); + + describe('cancelIfOutstanding', () => { + /** A complete report, so the fixture cannot drift from the contract cancelOrder now promises. */ + function cancelReport(overrides: Partial = {}): ScryptExecutionReport { + return { + ClOrdID: 'cancel-req-1', + OrigClOrdID: 'dfx-lm-7', + Symbol: 'EUR/USDT', + Side: 'Sell', + OrdStatus: ScryptOrderStatus.CANCELED, + OrderQty: '100', + CumQty: '0', + LeavesQty: '0', + ...overrides, + }; + } + + function stubCancel(report: ScryptExecutionReport | Error): void { + jest.spyOn(service as any, 'getTradePair').mockResolvedValue({ symbol: 'EUR/USDT' }); + const connection = (service as any).connection; + jest + .spyOn(connection, 'requestAndWaitForUpdate') + .mockImplementation(async () => (report instanceof Error ? Promise.reject(report) : report)); + } + + it('settles a cancellation that filled nothing', async () => { + stubCancel(cancelReport()); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe(ScryptCancellation.SETTLED); + }); + + it('reports a partial fill as executed — a terminal status does not mean nothing happened', async () => { + // the venue cancels a partially filled order with BOTH a terminal state and a non-zero filled size; + // reading only the state is how a real fill gets dropped + stubCancel(cancelReport({ CumQty: '40' })); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe(ScryptCancellation.EXECUTED); + }); + + it('settles a refusal that says there is no such order', async () => { + // a refused cancel arrives as an execution report, not as an error + stubCancel( + cancelReport({ OrdStatus: ScryptOrderStatus.NEW, ExecType: 'CancelRejected', CxlRejReason: 'UnknownOrder' }), + ); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe(ScryptCancellation.SETTLED); + }); + + it.each([ScryptOrderStatus.CANCELED, ScryptOrderStatus.FILLED])( + 'settles nothing when a %s report claims the order is unknown yet reports a fill', + async (ordStatus) => { + // the contradiction is the same whatever status rides along; deciding on the status first would let + // it through with a terminal one attached + stubCancel( + cancelReport({ + OrdStatus: ordStatus, + ExecType: 'CancelRejected', + CxlRejReason: 'UnknownOrder', + CumQty: '40', + }), + ); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe( + ScryptCancellation.UNCONFIRMED, + ); + }, + ); + + it.each(['', ' ', 'abc', undefined])('does not cache a cancellation whose filled size is %p', async (cumQty) => { + // readers derive the fill with `parseFloat(...) || 0`, so such an entry would quietly claim nothing + // was filled on every later lookup + stubCancel(cancelReport({ CumQty: cumQty as unknown as string })); + + await service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT'); + + expect((service as any).executionReports.has('dfx-lm-7')).toBe(false); + }); + + it('settles nothing when a refusal claims the order is unknown yet reports a fill', async () => { + // an order the venue has no record of cannot have traded — the report disagrees with itself, and + // nothing may be concluded from that, least of all that walking away is safe + stubCancel( + cancelReport({ + OrdStatus: ScryptOrderStatus.PARTIALLY_FILLED, + ExecType: 'CancelRejected', + CxlRejReason: 'UnknownOrder', + CumQty: '40', + LeavesQty: '60', + }), + ); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe( + ScryptCancellation.UNCONFIRMED, + ); + }); + + it('settles nothing on any other refusal — too late to cancel means it may yet execute', async () => { + stubCancel( + cancelReport({ OrdStatus: ScryptOrderStatus.NEW, ExecType: 'CancelRejected', CxlRejReason: 'TooLateToCancel' }), + ); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe( + ScryptCancellation.UNCONFIRMED, + ); + }); + + it.each([ + ['nothing filled', '0', ScryptCancellation.SETTLED], + ['a fill', '40', ScryptCancellation.EXECUTED], + ])('treats a rejected order with %s as terminal too', async (_label, cumQty, expected) => { + // a rejected order is as final as a cancelled one; a second opinion on what counts as terminal would + // be free to disagree with the first and leave such an order stuck + stubCancel(cancelReport({ OrdStatus: ScryptOrderStatus.REJECTED, CumQty: cumQty })); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe(expected); + }); + + it('settles nothing when a refused cancel reports a fill — the order is still open', async () => { + // a refusal carries the order's last known state, so a partially filled order that could NOT be + // cancelled reports a fill while remaining live. Reading the fill alone would call it finished and + // let the caller walk away from a reference that can still trade. + stubCancel( + cancelReport({ + OrdStatus: ScryptOrderStatus.PARTIALLY_FILLED, + ExecType: 'CancelRejected', + CxlRejReason: 'TooLateToCancel', + CumQty: '40', + LeavesQty: '60', + }), + ); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe( + ScryptCancellation.UNCONFIRMED, + ); + }); + + it('reports a fully filled order as executed', async () => { + stubCancel(cancelReport({ OrdStatus: ScryptOrderStatus.FILLED, CumQty: '100', LeavesQty: '0' })); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe(ScryptCancellation.EXECUTED); + }); + + it('settles nothing when the filled size cannot be read — that is not a zero', async () => { + stubCancel(cancelReport({ CumQty: undefined as unknown as string })); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe( + ScryptCancellation.UNCONFIRMED, + ); + }); + + it.each([ + ['', 'empty'], + [' ', 'whitespace'], + ])('settles nothing when the filled size is %p (%s) — that is missing, not zero', async (cumQty) => { + // Number('') is 0, not NaN, so this would otherwise pass a finite check and read as untouched + stubCancel(cancelReport({ CumQty: cumQty })); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe( + ScryptCancellation.UNCONFIRMED, + ); + }); + + it('settles nothing when the filled size is not a number at all', async () => { + stubCancel(cancelReport({ CumQty: 'abc' })); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe( + ScryptCancellation.UNCONFIRMED, + ); + }); + + it.each(['-1', '-0.5'])( + 'settles nothing when the filled size is %p — below zero is not untouched', + async (cumQty) => { + // A negative value is finite and parses cleanly, so it would pass every check above and then lose to + // `filled > 0` — reported as an order nothing ever traded. A cumulative filled size cannot be below + // zero, so this is a report not being understood, and misreading it grants exactly the false certainty + // that lets a live reference be walked away from. + stubCancel(cancelReport({ CumQty: cumQty })); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe( + ScryptCancellation.UNCONFIRMED, + ); + }, + ); + + it('waits past a PendingCancel report instead of taking it for the answer', async () => { + // the waiter resolves on its first match and then stops listening, so accepting the interim state + // would freeze it as the result and the real terminal report would never be seen. Exercises the + // matcher itself rather than mocking around it. + const connection = (service as any).connection; + let matcher: (reports: ScryptExecutionReport[]) => ScryptExecutionReport | null; + jest.spyOn(service as any, 'getTradePair').mockResolvedValue({ symbol: 'EUR/USDT' }); + jest.spyOn(connection, 'requestAndWaitForUpdate').mockImplementation(async (..._args: unknown[]) => { + matcher = _args[3] as typeof matcher; + return cancelReport(); + }); + + await service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT'); + + const pending = cancelReport({ OrdStatus: ScryptOrderStatus.PENDING_CANCEL }); + const terminal = cancelReport(); + expect(matcher([pending])).toBeNull(); + expect(matcher([pending, terminal])).toBe(terminal); + }); + + it('settles nothing when the cancel never came back', async () => { + stubCancel(new ScryptRequestTimeoutError('Timeout waiting for ExecutionReport update after 60000ms')); + + await expect(service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT')).resolves.toBe( + ScryptCancellation.UNCONFIRMED, + ); + }); + + it('files nothing when the cancel was refused — a refusal carries the last known state, not a verdict', async () => { + // for a reference the venue never had, that state reads as a live New order. Filing it would invent + // one, and the next lookup would call the order sent against a reference that never executed. + stubCancel( + cancelReport({ OrdStatus: ScryptOrderStatus.NEW, ExecType: 'CancelRejected', CxlRejReason: 'UnknownOrder' }), + ); + + await service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT'); + + expect((service as any).executionReports.has('dfx-lm-7')).toBe(false); + }); + + it('never files a cleanup cancellation under the order it cancelled', async () => { + // the confirmation carries the cancel request's id; filing it under the order would make a later + // status lookup read that order as terminally cancelled. A cleanup cancellation says nothing about + // the order as a whole — sibling references may still be unsettled and live — so a lookup reporting + // it as known would take it out of quarantine and let a replacement be opened beside them. + stubCancel(cancelReport({ CumQty: '40' })); + + await service.cancelIfOutstanding('dfx-lm-7', 'EUR', 'USDT'); + + expect((service as any).executionReports.has('dfx-lm-7')).toBe(false); + }); + }); + + describe('cancelIfOutstandingBySymbol', () => { + /** A complete report, so the fixture cannot drift from the contract cancelOrderBySymbol now promises. */ + function cancelReport(overrides: Partial = {}): ScryptExecutionReport { + return { + ClOrdID: 'cancel-req-2', + OrigClOrdID: 'legacy-ref-1', + Symbol: 'XRP/USDT', + Side: 'Sell', + OrdStatus: ScryptOrderStatus.CANCELED, + OrderQty: '10', + CumQty: '0', + LeavesQty: '0', + ...overrides, + }; + } + + it('cancels under the given symbol without deriving a trade pair', async () => { + const getTradePairSpy = jest.spyOn(service as any, 'getTradePair'); + const connection = (service as any).connection; + jest.spyOn(connection, 'requestAndWaitForUpdate').mockImplementation(async (..._args: unknown[]) => { + const [, [payload]] = _args as [unknown, [{ Symbol: string }]]; + expect(payload.Symbol).toBe('XRP/USDT'); + return cancelReport(); + }); + + await expect(service.cancelIfOutstandingBySymbol('legacy-ref-1', 'XRP/USDT')).resolves.toBe( + ScryptCancellation.SETTLED, + ); + expect(getTradePairSpy).not.toHaveBeenCalled(); + }); + + it('shares the same evaluation as cancelIfOutstanding — a fill is reported as executed here too', async () => { + const connection = (service as any).connection; + jest.spyOn(connection, 'requestAndWaitForUpdate').mockResolvedValue(cancelReport({ CumQty: '3' })); + + await expect(service.cancelIfOutstandingBySymbol('legacy-ref-1', 'XRP/USDT')).resolves.toBe( + ScryptCancellation.EXECUTED, + ); + }); + + it('goes unconfirmed and forgets the cached report the same way cancelIfOutstanding does', async () => { + const connection = (service as any).connection; + jest + .spyOn(connection, 'requestAndWaitForUpdate') + .mockRejectedValue(new ScryptRequestTimeoutError('Timeout waiting for ExecutionReport update after 60000ms')); + (service as any).executionReports.set('legacy-ref-1', { + ClOrdID: 'legacy-ref-1', + OrdStatus: ScryptOrderStatus.NEW, + }); + + await expect(service.cancelIfOutstandingBySymbol('legacy-ref-1', 'XRP/USDT')).resolves.toBe( + ScryptCancellation.UNCONFIRMED, + ); + expect((service as any).executionReports.has('legacy-ref-1')).toBe(false); + }); + + it('settles a refusal that says there is no such order — same UnknownOrder inference as cancelIfOutstanding', async () => { + const connection = (service as any).connection; + jest + .spyOn(connection, 'requestAndWaitForUpdate') + .mockResolvedValue( + cancelReport({ OrdStatus: ScryptOrderStatus.NEW, ExecType: 'CancelRejected', CxlRejReason: 'UnknownOrder' }), + ); + + await expect(service.cancelIfOutstandingBySymbol('legacy-ref-1', 'XRP/USDT')).resolves.toBe( + ScryptCancellation.SETTLED, + ); + }); + }); + describe('checkTrade — the amend write boundary', () => { function stubAmendPath(editOutcome: Error): void { jest.spyOn(service as any, 'getOrderStatus').mockResolvedValue({ @@ -816,7 +1330,16 @@ describe('ScryptService', () => { }); jest.spyOn(service as any, 'getTradePrice').mockResolvedValue(2); jest.spyOn(service as any, 'editOrder').mockRejectedValue(editOutcome); - jest.spyOn(service as any, 'cancelOrder').mockResolvedValue(undefined); + jest.spyOn(service as any, 'cancelOrder').mockResolvedValue({ + ClOrdID: 'cancel-req-1', + OrigClOrdID: 'dfx-lm-7', + Symbol: 'EUR/USDT', + Side: 'Sell', + OrdStatus: ScryptOrderStatus.CANCELED, + OrderQty: '5', + CumQty: '0', + LeavesQty: '0', + } satisfies ScryptExecutionReport); } it('propagates an unconfirmed amend instead of swallowing it', async () => { @@ -864,28 +1387,239 @@ describe('ScryptService', () => { expect((service as any).executionReports.has('dfx-lm-7')).toBe(true); }); - it('keeps waiting on a pending order however old it is — pending is observed, not unknown', async () => { - // quarantining it would make reconciliation find the reference, hand the order back, and the next - // completion check quarantine it again: a loop, not a resolution + it('keeps waiting on a pending order that is still young, without asking Scrypt to cancel it', async () => { + // without this, a fresh PENDING report would trigger a cancel-and-decide it has not earned yet — the + // bound exists precisely so a normal, seconds-long transition is never treated as stuck + jest.spyOn(service as any, 'getOrderStatus').mockResolvedValue({ + id: 'dfx-lm-7', + status: ScryptOrderStatus.PENDING_NEW, + remainingQuantity: 5, + }); + const cancelSpy = jest.spyOn(service, 'cancelIfOutstanding'); + + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', new Date())).resolves.toBe(false); + expect(cancelSpy).not.toHaveBeenCalled(); + }); + + it('keeps waiting when an old order has only just entered a pending state', async () => { + // Without the dwell-time clock, a price adjustment on an hours-old healthy order would cancel it + // because the old logic measured order age instead of time spent pending. + // + // Two passes on purpose: the FIRST pending poll only records the observation and returns, whatever + // the bound is measured against, so asserting it alone would pass just as happily against the age + // clock this replaced. The second pass is where the two differ — 120 minutes of age is past the + // bound, seconds of dwell time are not — so only this one actually pins the fix down. + jest.spyOn(service as any, 'getOrderStatus').mockResolvedValue({ + id: 'dfx-lm-7', + status: ScryptOrderStatus.PENDING_REPLACE, + remainingQuantity: 5, + }); + const cancelSpy = jest.spyOn(service, 'cancelIfOutstanding'); + const orderCreated = new Date(Date.now() - 120 * 60 * 1000); + + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', orderCreated)).resolves.toBe(false); + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', orderCreated)).resolves.toBe(false); + + expect(cancelSpy).not.toHaveBeenCalled(); + }); + + it('starts a fresh pending clock when a reference leaves pending before re-entering it', async () => { + // Without clearing pendingSince on the intervening non-pending status, a later PENDING entry would + // inherit the old, potentially expired wait instead of starting a new observation period. + jest + .spyOn(service as any, 'getOrderStatus') + .mockResolvedValueOnce({ + id: 'dfx-lm-7', + status: ScryptOrderStatus.PENDING_NEW, + remainingQuantity: 5, + }) + .mockResolvedValueOnce({ + id: 'dfx-lm-7', + status: ScryptOrderStatus.FILLED, + remainingQuantity: 0, + }) + .mockResolvedValueOnce({ + id: 'dfx-lm-7', + status: ScryptOrderStatus.PENDING_NEW, + remainingQuantity: 5, + }); + const cancelSpy = jest.spyOn(service, 'cancelIfOutstanding'); + + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', new Date())).resolves.toBe(false); + (service as any).pendingSince.set('dfx-lm-7', { + since: new Date(Date.now() - 6 * 60 * 1000), + lastSeen: new Date(Date.now() - 6 * 60 * 1000), + }); + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', new Date())).resolves.toBe(true); + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', new Date())).resolves.toBe(false); + + expect(cancelSpy).not.toHaveBeenCalled(); + }); + + it('keeps a reference that is still being observed, however long it has been pending', async () => { + // The prune sweep runs on lastSeen, never on since. A reference the venue has reported pending for + // more than a day is not stale — it is still polled every pass and retried under the cancel throttle. + // Pruning it by `since` would drop a live entry and hand it a fresh five-minute grace period, so the + // stuck order would never reach its bound at all. jest.spyOn(service as any, 'getOrderStatus').mockResolvedValue({ id: 'dfx-lm-7', status: ScryptOrderStatus.PENDING_NEW, remainingQuantity: 5, }); + const orderCreated = new Date(Date.now() - 30 * 60 * 60 * 1000); + + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', orderCreated)).resolves.toBe(false); + + // Age BOTH stamps, then observe the reference again. That second pass is the point: it takes the + // known-entry branch, which is the only place lastSeen is refreshed. Without asserting it here, that + // refresh could be deleted outright and the sweep below would still find the stamp the first pass + // wrote — the test would pass while the entry silently aged into being prunable. + const staleAt = new Date(Date.now() - 25 * 60 * 60 * 1000); + (service as any).pendingSince.set('dfx-lm-7', { since: staleAt, lastSeen: staleAt }); + // a day past its bound, so this pass reaches the cancel — the venue not confirming it is what keeps + // such a reference pending indefinitely, which is exactly the case the sweep must not collect + jest.spyOn(service, 'cancelIfOutstanding').mockResolvedValue(ScryptCancellation.UNCONFIRMED); + + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', orderCreated)).resolves.toBe(false); + expect((service as any).pendingSince.get('dfx-lm-7').lastSeen.getTime()).toBeGreaterThan(staleAt.getTime()); + // the clock itself must NOT be pushed forward, or the bound would never be reached + expect((service as any).pendingSince.get('dfx-lm-7').since).toEqual(staleAt); + + // a different reference entering pending for the first time is what triggers the sweep + jest.spyOn(service as any, 'getOrderStatus').mockResolvedValue({ + id: 'dfx-lm-8', + status: ScryptOrderStatus.PENDING_NEW, + remainingQuantity: 5, + }); + await expect(service.checkTrade('dfx-lm-8', 'EUR', 'USDT', orderCreated)).resolves.toBe(false); + + expect((service as any).pendingSince.has('dfx-lm-7')).toBe(true); + }); + + it('keeps waiting on a pending order past its bound when Scrypt will not confirm a cancel', async () => { + // the most important case here: without a confirmed cancel the order may still be live in the book, and + // giving it up on unconfirmed evidence could let the onFail chain place a second, genuinely competing buy; + // resolving false alone would not distinguish waiting after an unconfirmed cancel from never trying one + jest.spyOn(service as any, 'getOrderStatus').mockResolvedValue({ + id: 'dfx-lm-7', + status: ScryptOrderStatus.PENDING_NEW, + remainingQuantity: 5, + }); + const cancelSpy = jest.spyOn(service, 'cancelIfOutstanding').mockResolvedValue(ScryptCancellation.UNCONFIRMED); + + // PENDING_STUCK_AFTER_MINUTES is private, so backdate the service's observation clock directly instead + // of driving Date.now(); only the pending-dwell input needs to be past its bound in this test. + (service as any).pendingSince.set('dfx-lm-7', { + since: new Date(Date.now() - 6 * 60 * 1000), + lastSeen: new Date(Date.now() - 6 * 60 * 1000), + }); await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', new Date(Date.now() - 120 * 60 * 1000))).resolves.toBe( false, ); + expect(cancelSpy).toHaveBeenCalledTimes(1); }); - it('keeps waiting on a pending order that is still young', async () => { + it('fails a pending order past its bound once Scrypt confirms nothing can execute under it any more', async () => { + // without this, a trade stuck reporting PENDING forever would have no exit but a human noticing jest.spyOn(service as any, 'getOrderStatus').mockResolvedValue({ id: 'dfx-lm-7', status: ScryptOrderStatus.PENDING_NEW, remainingQuantity: 5, }); + jest.spyOn(service, 'cancelIfOutstanding').mockResolvedValue(ScryptCancellation.SETTLED); - await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', new Date())).resolves.toBe(false); + // PENDING_STUCK_AFTER_MINUTES is private, so backdate the service's observation clock directly instead + // of driving Date.now(); only the pending-dwell input needs to be past its bound in this test. + (service as any).pendingSince.set('dfx-lm-7', { + since: new Date(Date.now() - 6 * 60 * 1000), + lastSeen: new Date(Date.now() - 6 * 60 * 1000), + }); + + await expect( + service.checkTrade('dfx-lm-7', 'EUR', 'USDT', new Date(Date.now() - 6 * 60 * 1000)), + ).rejects.toBeInstanceOf(ScryptOrderStuckPendingError); + expect((service as any).pendingSince.has('dfx-lm-7')).toBe(false); + }); + + it('completes a pending order past its bound when Scrypt confirms it filled after all', async () => { + // without this, a trade that actually filled would be failed instead of counted as complete + jest.spyOn(service as any, 'getOrderStatus').mockResolvedValue({ + id: 'dfx-lm-7', + status: ScryptOrderStatus.PENDING_NEW, + remainingQuantity: 5, + }); + jest.spyOn(service, 'cancelIfOutstanding').mockResolvedValue(ScryptCancellation.EXECUTED); + + // PENDING_STUCK_AFTER_MINUTES is private, so backdate the service's observation clock directly instead + // of driving Date.now(); only the pending-dwell input needs to be past its bound in this test. + (service as any).pendingSince.set('dfx-lm-7', { + since: new Date(Date.now() - 6 * 60 * 1000), + lastSeen: new Date(Date.now() - 6 * 60 * 1000), + }); + + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', new Date(Date.now() - 6 * 60 * 1000))).resolves.toBe( + true, + ); + expect((service as any).pendingSince.has('dfx-lm-7')).toBe(false); + }); + + it('does not retry the cancel write on the very next pass after an unconfirmed attempt', async () => { + // the cancel is a WRITE and checkRunningOrders runs nominally every 10 seconds, and possibly more often + // within one pass — without a retry floor per reference this would fire a fresh cancel on every one of + // those calls for as long as the venue keeps not confirming it + jest.spyOn(service as any, 'getOrderStatus').mockResolvedValue({ + id: 'dfx-lm-7', + status: ScryptOrderStatus.PENDING_NEW, + remainingQuantity: 5, + }); + const cancelSpy = jest.spyOn(service, 'cancelIfOutstanding').mockResolvedValue(ScryptCancellation.UNCONFIRMED); + const orderCreated = new Date(Date.now() - 6 * 60 * 1000); + + // PENDING_STUCK_AFTER_MINUTES is private, so backdate the service's observation clock directly instead + // of driving Date.now(); only the pending-dwell input needs to be past its bound in this test. + (service as any).pendingSince.set('dfx-lm-7', { + since: new Date(Date.now() - 6 * 60 * 1000), + lastSeen: new Date(Date.now() - 6 * 60 * 1000), + }); + + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', orderCreated)).resolves.toBe(false); + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', orderCreated)).resolves.toBe(false); + + expect(cancelSpy).toHaveBeenCalledTimes(1); + }); + + it('retries the cancel write again once the retry floor has passed', async () => { + // Mirrors the previous test but advances past PENDING_CANCEL_RETRY_MINUTES between passes: without + // this, an implementation whose retry floor never expires would look identical to a correct one to + // the test above alone, and a permanently blocked cancel is itself a new stuck-forever wait state. + jest.spyOn(service as any, 'getOrderStatus').mockResolvedValue({ + id: 'dfx-lm-7', + status: ScryptOrderStatus.PENDING_NEW, + remainingQuantity: 5, + }); + const cancelSpy = jest.spyOn(service, 'cancelIfOutstanding').mockResolvedValue(ScryptCancellation.UNCONFIRMED); + const orderCreated = new Date(Date.now() - 6 * 60 * 1000); + + // PENDING_STUCK_AFTER_MINUTES is private, so backdate the service's observation clock directly instead + // of driving Date.now(); only the pending-dwell input needs to be past its bound in this test. + (service as any).pendingSince.set('dfx-lm-7', { + since: new Date(Date.now() - 6 * 60 * 1000), + lastSeen: new Date(Date.now() - 6 * 60 * 1000), + }); + + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', orderCreated)).resolves.toBe(false); + expect(cancelSpy).toHaveBeenCalledTimes(1); + + // PENDING_CANCEL_RETRY_MINUTES is a private module-level constant (1 minute) with no export to + // import in this test, so the recorded attempt is backdated directly on the service's private map + // instead of driving Date.now() itself — this is the one throttle input the test needs to move, and + // moving only it keeps the assertion tied to the retry floor, not incidentally to the pending dwell. + const pastRetryFloor = new Date(Date.now() - 2 * 60 * 1000); + (service as any).pendingCancelAttempts.set('dfx-lm-7', pastRetryFloor); + + await expect(service.checkTrade('dfx-lm-7', 'EUR', 'USDT', orderCreated)).resolves.toBe(false); + expect(cancelSpy).toHaveBeenCalledTimes(2); }); it('cancels on an explicit rejection, but reports the refusal and the spent reference', async () => { diff --git a/src/integration/exchange/services/scrypt-websocket-connection.ts b/src/integration/exchange/services/scrypt-websocket-connection.ts index 089a7527a3..e1da99de52 100644 --- a/src/integration/exchange/services/scrypt-websocket-connection.ts +++ b/src/integration/exchange/services/scrypt-websocket-connection.ts @@ -113,6 +113,31 @@ export class ScryptUnconfirmedWriteError extends Error { */ export class ScryptOrderNotFoundError extends Error {} +/** + * A trade whose venue reference this process has continuously observed reporting PENDING_NEW/PENDING_CANCEL/ + * PENDING_REPLACE for longer than PENDING_STUCK_AFTER_MINUTES, and whose outstanding reference the venue then + * answered on an explicit cancel request with `ScryptCancellation.SETTLED` — one of two distinct qualities of + * answer, per that enum's own documentation: a terminal cancel with nothing filled, or the venue not + * recognising the reference at all (`SCRYPT_UNKNOWN_ORDER`), which is an inference from its own words rather + * than a statement about execution. + * + * The bound measures how long this process has continuously observed the reference as PENDING, beginning + * with its first such observation. It does not measure the age of the order itself. + * + * Both qualities of SETTLED carry the weight this error rests on, but not for the same reason. A terminal + * cancel states it outright. `UnknownOrder` rests on the inference documented at SCRYPT_UNKNOWN_ORDER: a + * venue that does not recognise a reference cannot execute anything under it. Deliberately NOT argued as + * the venue contradicting its own last answer — the status read here comes from the cached execution report + * and is not re-fetched before the cancel, so a live push may have moved it on in between, and "its last + * answer" is then not what this branch saw. + * + * Distinct from {@link ScryptOrderNotFoundError}: that one means the venue cannot find the order at all, an + * unresolved blind spot. Here the venue found it, answered PENDING, and then settled it on request — which is + * why the caller may fail the order instead of quarantining it. No change in behaviour from this wording, + * only a more honest account of what SETTLED actually rests on. + */ +export class ScryptOrderStuckPendingError extends Error {} + /** * An amend the venue refused. The replacement was never created, so the ORIGINAL order is still live — and * its reference is spent, because the venue requires references to be unique. Carries it so the caller can diff --git a/src/integration/exchange/services/scrypt.service.ts b/src/integration/exchange/services/scrypt.service.ts index 9c57bd4049..ee07e4c992 100644 --- a/src/integration/exchange/services/scrypt.service.ts +++ b/src/integration/exchange/services/scrypt.service.ts @@ -9,6 +9,7 @@ import { PricingProvider } from 'src/subdomains/supporting/pricing/services/inte import { ScryptBalance, ScryptBalanceTransaction, + ScryptCancellation, ScryptDepositStatus, ScryptExecutionReport, ScryptMarketDataSnapshot, @@ -25,6 +26,7 @@ import { ScryptTransactionType, ScryptWithdrawResponse, ScryptWithdrawStatus, + isTerminalScryptOrderStatus, } from '../dto/scrypt.dto'; import { TradeChangedException } from '../exceptions/trade-changed.exception'; import { @@ -33,6 +35,7 @@ import { ScryptAmendRejectedError, ScryptMessageType, ScryptOrderNotFoundError, + ScryptOrderStuckPendingError, ScryptUnconfirmedWriteError, ScryptVenueRejectionError, ScryptWebSocketConnection, @@ -40,9 +43,72 @@ import { /** * After this long without a usable answer, an order the venue once acknowledged is treated as lost rather - * than merely slow. Shared by the "cannot be found" and the "stuck pending" paths so both give up together. + * than merely slow. One use only: the branch below where the status lookup returns nothing at all. The + * pending states deliberately do NOT consult it — a venue that still reports PENDING_NEW is answering, and + * this constant is about silence. They are bounded separately by PENDING_STUCK_AFTER_MINUTES, which asks + * for a cancel rather than declaring the order lost, because there the reference is known to exist. + * + * Kept equal to SCRYPT_UNOBSERVABLE_QUARANTINE_MINUTES in the adapter, which documents itself as matching + * this value — the two are the pair of routes out of a silent order and must not drift apart. Five rather + * than sixty so that quarantine plus the abandon bound stay inside the ten-minute ceiling. Safe at this + * length because it only fires while the venue does not know the reference at all: an order it is still + * working is returned by the status lookup and never reaches here, whatever its age. Measured over 60 days, + * Scrypt trades complete in a median of 0.2 and a maximum of 1.0 minutes. */ -const ORDER_LOST_AFTER_MINUTES = 60; +const ORDER_LOST_AFTER_MINUTES = 5; + +/** + * PENDING_NEW / PENDING_CANCEL / PENDING_REPLACE are meant to be a transition lasting seconds — the venue + * is in the middle of accepting, cancelling or replacing the order. Measured over 60 days, Scrypt trades + * spend a median of 0.2 and a maximum of 1.0 minutes reaching a terminal or open state, so a reference that + * has continuously answered PENDING for more than five minutes is not merely slow, it is stuck. + * + * Measured from when THIS process first observes the venue continuously reporting a PENDING state for the + * reference, not from the order's creation. A process restart starts that observation clock afresh; this can + * only extend the bound, never cause the adapter to give up too early. + * + * Deliberately its own constant rather than reusing ORDER_LOST_AFTER_MINUTES, even though both happen to be + * five: that one bounds SILENCE from the venue (no status at all), this one bounds an ANSWER that makes no + * progress. Merging them would let a later change meant for one quietly shift the other along with it. + * + * The bound never ends the order by itself — the exit is canceling and restarting, never a bare give-up: a + * trade that might still be sitting in the book must not be abandoned unconfirmed, or the onFail chain could + * place a second, genuinely competing buy alongside it. + */ +const PENDING_STUCK_AFTER_MINUTES = 5; + +/** + * Minimum wait between two cancel attempts for the SAME pending reference, once it is past its bound. + * + * The cancel below is a WRITE to the venue, and `checkRunningOrders` is driven by a cron that runs nominally + * every ten seconds, and possibly more often within one pass — the cron itself has a lock, a jitter lead-in + * and a process-disable gate, and `processPipelines`'s own `while (hasChanges)` loop can invoke it again + * before the next tick. Without a floor here, an UNCONFIRMED answer would draw a fresh cancel on every one of + * those calls for as long as the venue keeps not confirming it. One minute, matching the cooldown floor the + * analogous quarantine-cancel throttle uses (`UNCERTAIN_RESOLVE_MIN_INTERVAL_MS` in + * liquidity-management-pipeline.service.ts), for the same stated reason: "a cancellation the venue will not + * confirm must not retry on every ten-second tick." This only slows the write down — it does not move + * PENDING_STUCK_AFTER_MINUTES itself, so the order becomes eligible for a cancel after exactly the same + * pending dwell time either way; only how often an unconfirmed attempt may repeat changes. + */ +const PENDING_CANCEL_RETRY_MINUTES = 1; + +// The venue answers a refused cancel with an execution report rather than a separate reject message, so the +// refusal has to be read off these two fields. `UnknownOrder` is the one reason treated as settling +// anything. +// +// That reading is an inference, not a documented guarantee — the protocol spec lists the reason without +// defining it, so "never existed" cannot be distinguished from "not processed yet" from the value alone. +// What it rests on: the caller only cancels an order after either a failed status lookup has outlived the +// full ORDER_LOST_AFTER_MINUTES window in which its request could still be in flight, or a successful lookup +// has continuously reported PENDING for the full PENDING_STUCK_AFTER_MINUTES window. The latter premise is +// even stronger: the venue just confirmed that the reference existed, then no longer knew it at the cancel — +// a stronger signal than silence alone. Note what that does and does not cover — the lookup stops at the +// first reference the venue does not show, so for every other reference of the same order this refusal is the +// only negative answer there is. Age plus one refusal is the strongest evidence this protocol offers. Every +// other reason (too late, rate limited, already pending) settles nothing and is waited out. +const SCRYPT_CANCEL_REJECTED = 'CancelRejected'; +const SCRYPT_UNKNOWN_ORDER = 'UnknownOrder'; // The bulk streams a reconnect catch-up restores; the live subscriptions cover everything else. type CatchUpStream = ScryptMessageType.EXECUTION_REPORT | ScryptMessageType.BALANCE_TRANSACTION; @@ -57,6 +123,13 @@ export class ScryptService extends PricingProvider { private readonly balances?: AsyncSubscription>; private readonly executionReports: Map = new Map(); private readonly balanceTransactions: Map = new Map(); + // Throttle for the WRITE in the PENDING branch of checkTrade — see PENDING_CANCEL_RETRY_MINUTES. Keyed by + // clOrdId, value is the end of the last cancel attempt for that reference. + private readonly pendingCancelAttempts: Map = new Map(); + // Tracks how long THIS process has continuously seen a reference report a PENDING status — see + // PENDING_STUCK_AFTER_MINUTES. `since` measures that bound; `lastSeen` is only for 24-hour cleanup. + // Cleared once the reference leaves the pending states, so a later re-entry starts fresh. + private readonly pendingSince: Map = new Map(); private catchUpInProgress = false; private catchUpPending = false; private lastCatchUpAt?: number; @@ -162,9 +235,23 @@ export class ScryptService extends PricingProvider { // A warm-up that loaded BOTH streams is exactly what a catch-up round does, so it claims the first slot and a // reconnect right after boot waits it out instead of repeating it. If either leg failed the caches are not - // whole, and the next reconnect must repair immediately rather than sit out the interval on stale state. + // whole: without an immediate retry the only refill path is onReconnect, so a stable socket after a failed + // warm-up would leave balanceTransactions / executionReports empty forever. An empty cache no longer blocks + // confirmWithdrawalAbsent (absence is decided from the fresh venue reply + live recheck only), but findWithdrawal + // and that live recheck still benefit from a filled cache, so incomplete boot warm-up must still be retried + // promptly rather than waiting for a human or a later reconnect. Reuse catchUpAfterReconnect (both streams, + // existing pacing / retry) rather than inventing a second timer type; trigger is only a true Promise rejection + // on a leg, never "empty array" (that is a successful warm-up with no rows). void Promise.all([executionWarmUp, balanceWarmUp]).then(([executionLoaded, balanceLoaded]) => { - if (executionLoaded && balanceLoaded) this.lastCatchUpAt = Date.now(); + if (executionLoaded && balanceLoaded) { + this.lastCatchUpAt = Date.now(); + return; + } + + this.logger.warn( + `Scrypt boot warm-up incomplete (executionLoaded=${executionLoaded}, balanceLoaded=${balanceLoaded}) — triggering catch-up without waiting for a reconnect`, + ); + void this.catchUpAfterReconnect().catch((e) => this.logger.error('Scrypt catch-up retry failed:', e)); }); this.connection.onReconnect(() => this.catchUpAfterReconnect()); @@ -187,7 +274,7 @@ export class ScryptService extends PricingProvider { } private isTerminalExecutionReport(r: ScryptExecutionReport): boolean { - return [ScryptOrderStatus.FILLED, ScryptOrderStatus.CANCELED, ScryptOrderStatus.REJECTED].includes(r.OrdStatus); + return isTerminalScryptOrderStatus(r.OrdStatus); } /** @@ -215,7 +302,17 @@ export class ScryptService extends PricingProvider { // Bulk (age-bounded) warm-up/catch-up path only — live subscriptions must cache directly via cacheExecutionReport/cacheBalanceTransaction, see constructor. private applyBalanceTransactions(transactions: ScryptBalanceTransaction[]): void { const cacheMaxAge = Util.daysBefore(365); - for (const t of transactions) if (new Date(t.Timestamp) >= cacheMaxAge) this.cacheBalanceTransaction(t); + for (const t of transactions) { + // Field priority matches the rest of this file: Timestamp first, TransactTime only when Timestamp is missing. + // Missing or unreadable stamp → cache conservatively (never drop). The bulk age filter must not discard a + // withdrawal we later need for findWithdrawal or the live recheck in confirmWithdrawalAbsent — a dropped + // row is a payout we cannot rediscover. This is a deliberate, documented fallback (cache on doubt), not a + // silent default. + const raw = t.Timestamp ?? t.TransactTime; + if (!raw || Number.isNaN(new Date(raw).getTime()) || new Date(raw) >= cacheMaxAge) { + this.cacheBalanceTransaction(t); + } + } } // After a WS reconnect, re-fetch balance transactions + execution reports so an event missed during the outage @@ -585,6 +682,75 @@ export class ScryptService extends PricingProvider { return found; } + /** + * Confirm that the venue's transaction history has no record of this withdrawal reference. + * + * Scrypt has no cancel/storno for withdrawals. A quarantined withdrawal therefore cannot be cleared the way + * a trade is. The automatic exit is confirmed absence: a fresh bulk fetch was returned and this `clReqId` + * is not in it — whether that history has rows or not. The caller abandons on that basis rather than + * claiming a not-sent release. + * + * No local consistency gate / cache-anchor check. Scrypt withdrawal destinations are exclusively DFX-owned + * addresses ("Auszahlungsadressen bei Scrypt gehören alle ausnahmslos der DFX AG"). A second payout would + * move funds only between DFX accounts — an internal rebooking, not a loss and not a compliance incident + * ("eine doppelte Auszahlung wäre absolut akzeptabel"). The former gates traded that accepted non-risk for a + * forbidden permanent wait: they required cache anchors that can be structurally absent (no row with + * ClReqID, no row older than the order, 365-day cache bound), so confirmWithdrawalAbsent returned false + * forever and the order waited on a human to refill the cache. "Die Kombination aus Scrypt und Warten auf + * einen Menschen ist NICHT ERLAUBT." + * + * Incomplete or truncated venue replies (pagination cut-off, partial answer) can now cause a second + * withdrawal. That is the deliberate trade-off accepted by the orderer — not an overlooked gap. + * + * Remaining `false` branches react only to a fetch failure or a live-cache race hit (the reference appeared + * in the live map while the bulk fetch was in flight). An empty history is no longer one of them: on a + * fresh or long-dormant Scrypt account the trade history genuinely has no rows, and a successful `[]` is + * treated exactly like any other successful reply that does not name the reference — not as a reason to + * wait. It is not evidence the history was complete: a truncated reply can arrive this way too, which is + * the trade-off stated above, accepted rather than overlooked. That is still not the same case as a real + * outage: an outage does not come back as an empty array, it throws, and lands in the catch above. The two + * look alike only on paper — a thrown error and a successful empty reply are deliberately handled + * differently. + * + * @returns true when the live cache does not hold `clReqId` after the fetch and the fresh history (empty or + * not) does not contain it. false on fetch failure, live-race hit, or when the reference is present in the + * fresh reply. + */ + async confirmWithdrawalAbsent(clReqId: string): Promise { + let fresh: ScryptBalanceTransaction[]; + try { + fresh = await this.connection.fetchAll(ScryptMessageType.BALANCE_TRANSACTION); + } catch (e) { + this.logger.warn(`confirmWithdrawalAbsent(${clReqId}): could not fetch full transaction history: ${e.message}`); + return false; + } + + // An empty reply is a successful answer that does not name the reference — not a reason to wait. See + // the JSDoc above for why this is no longer a `false` branch: it carries exactly the weight of a + // non-empty history without `clReqId` below, no more, and is not proof the history was complete. + if (!fresh.length) { + this.logger.info( + `confirmWithdrawalAbsent(${clReqId}): venue returned an empty transaction history — the reference cannot exist in a history with no rows at all`, + ); + } + + const freshIds = new Set(fresh.map((t) => t.ClReqID).filter((id): id is string => Boolean(id))); + + // A live subscription (or catch-up) can write clReqId into this.balanceTransactions while the bulk fetch + // is still open — freshIds then misses it, and returning true would let the caller abandon and replan a + // second withdrawal. Only a re-read of the LIVE map after the await sees that race; if the id is there, + // absence is not confirmed (false, not a hard failure). This is the one remaining positive observation + // that can still block absence confirmation without requiring a local cache anchor. + if (this.balanceTransactions.has(clReqId)) { + this.logger.warn( + `confirmWithdrawalAbsent(${clReqId}): reference appeared in the live cache while the history fetch was in flight — cannot conclude absence`, + ); + return false; + } + + return !freshIds.has(clReqId); + } + /** * @param since lower bound for the fallback history fetch. A caller that knows when its reference can * earliest have existed passes it here, so a lookup for an absent order does not pull a full 30 days of @@ -624,6 +790,17 @@ export class ScryptService extends PricingProvider { }; } + // Prunes on every write rather than on a timer or a terminal event: this throttle only ever needs the MOST + // RECENT attempt, so nothing is lost by dropping an old one, and 24 hours is far beyond + // PENDING_STUCK_AFTER_MINUTES (single-digit minutes) — an entry that old belongs to a reference that has + // long since resolved or moved on, not one still cycling through the PENDING branch below. + private recordPendingCancelAttempt(clOrdId: string): void { + this.pendingCancelAttempts.set(clOrdId, new Date()); + + const dayAgo = Util.hoursBefore(24); + for (const [id, at] of this.pendingCancelAttempts) if (at < dayAgo) this.pendingCancelAttempts.delete(id); + } + /** * @param replacementClOrdId reference to use if this check has to amend or restart the order. Must be * reproducible from the order row by the caller, so a timed-out replacement stays findable. @@ -632,7 +809,7 @@ export class ScryptService extends PricingProvider { clOrdId: string, from: string, to: string, - orderCreated?: Date, + orderCreated: Date, replacementClOrdId?: string, // Invoked immediately before a replacement is sent, so the caller can make the reference durable first. // Without that, a replacement whose confirmation is lost is neither the current reference nor a spent @@ -641,8 +818,8 @@ export class ScryptService extends PricingProvider { ): Promise { const orderInfo = await this.getOrderStatus(clOrdId); if (!orderInfo) { - // If the order is older than 1 hour and still not found, it's lost - const ageMinutes = orderCreated ? Util.minutesDiff(orderCreated) : 0; + // Past its bound and still not found anywhere: treat it as lost rather than keep polling for it. + const ageMinutes = Util.minutesDiff(orderCreated); if (ageMinutes > ORDER_LOST_AFTER_MINUTES) { throw new ScryptOrderNotFoundError( `Order ${clOrdId} not found after ${Math.round(ageMinutes)} minutes — it may have completed or been cancelled outside of tracked state`, @@ -653,6 +830,13 @@ export class ScryptService extends PricingProvider { return false; } + if ( + orderInfo.status !== ScryptOrderStatus.PENDING_NEW && + orderInfo.status !== ScryptOrderStatus.PENDING_CANCEL && + orderInfo.status !== ScryptOrderStatus.PENDING_REPLACE + ) + this.pendingSince.delete(clOrdId); + switch (orderInfo.status) { case ScryptOrderStatus.NEW: case ScryptOrderStatus.PARTIALLY_FILLED: { @@ -692,7 +876,8 @@ export class ScryptService extends PricingProvider { this.logger.verbose(`Could not update order ${clOrdId}, attempting cancel: ${e.message}`); let cancelConfirmed = true; try { - await this.cancelOrder(clOrdId, from, to); + const cancelReport = await this.cancelOrder(clOrdId, from, to); + cancelConfirmed = cancelReport.OrdStatus === ScryptOrderStatus.CANCELED; } catch (cancelError) { // The cancel is a write too. Unconfirmed, it may well have taken effect at the venue while the // cached report still shows the order open — and a non-terminal entry is never refreshed, so @@ -771,14 +956,81 @@ export class ScryptService extends PricingProvider { case ScryptOrderStatus.PENDING_NEW: case ScryptOrderStatus.PENDING_CANCEL: - case ScryptOrderStatus.PENDING_REPLACE: - // Deliberately just waits, however old the order is. A pending report is an OBSERVATION — we know - // where the order stands — so it is not an unknown outcome and must not be quarantined: reconciliation - // would find the reference, hand the order straight back, and the next completion check would - // quarantine it again. An order that stays pending too long is a stuck order, which the monitoring - // counter surfaces; it is not an unresolved one. - this.logger.verbose(`Order ${clOrdId} is pending (${orderInfo.status}), waiting...`); + case ScryptOrderStatus.PENDING_REPLACE: { + // A pending report is an OBSERVATION — we know where the order stands — so on its own it is not an + // unknown outcome and must not be quarantined: reconciliation would find the reference, hand the + // order straight back, and the next completion check would quarantine it again. But "observed" is not + // the same claim as "running": PENDING_* is meant to last seconds (see PENDING_STUCK_AFTER_MINUTES), + // so past that bound this stops merely waiting and asks the venue to settle the question. The clock + // alone is never the exit, though — only a confirmed cancel is: a trade that might still be sitting in + // the book must not be given up on unconfirmed evidence, or the onFail chain could place a second, + // genuinely competing buy right next to it. That cancel is a WRITE to the venue, so it is throttled to + // at most one attempt per PENDING_CANCEL_RETRY_MINUTES per reference — otherwise an UNCONFIRMED answer + // would draw a fresh cancel on every checkRunningOrders call, which runs nominally every ten seconds, + // and possibly more often within one pass. + const pendingEntry = this.pendingSince.get(clOrdId); + if (!pendingEntry) { + const now = new Date(); + this.pendingSince.set(clOrdId, { since: now, lastSeen: now }); + + // Clean up by `lastSeen`, not `since`: a reference may remain continuously pending indefinitely while + // still being observed and retried by the PENDING_CANCEL_RETRY_MINUTES throttle below. Cleaning by + // `since` would drop an active reference and incorrectly grant a new grace period on its next tick. + const dayAgo = Util.hoursBefore(24); + for (const [id, entry] of this.pendingSince) if (entry.lastSeen < dayAgo) this.pendingSince.delete(id); + + this.logger.verbose(`Order ${clOrdId} is pending (${orderInfo.status}), waiting...`); + return false; + } + + pendingEntry.lastSeen = new Date(); + const pendingMinutes = Util.minutesDiff(pendingEntry.since); + if (pendingMinutes <= PENDING_STUCK_AFTER_MINUTES) { + this.logger.verbose(`Order ${clOrdId} is pending (${orderInfo.status}), waiting...`); + return false; + } + + const lastCancelAttempt = this.pendingCancelAttempts.get(clOrdId); + if (lastCancelAttempt && Util.minutesDiff(lastCancelAttempt) < PENDING_CANCEL_RETRY_MINUTES) { + this.logger.verbose( + `Order ${clOrdId} is pending (${orderInfo.status}) past its bound, but its last cancel attempt was ` + + `less than ${PENDING_CANCEL_RETRY_MINUTES} minute(s) ago — waiting before retrying the write`, + ); + return false; + } + + let cancellation: ScryptCancellation; + try { + cancellation = await this.cancelIfOutstanding(clOrdId, from, to); + } finally { + this.recordPendingCancelAttempt(clOrdId); + } + + if (cancellation === ScryptCancellation.EXECUTED) { + this.pendingCancelAttempts.delete(clOrdId); + this.pendingSince.delete(clOrdId); + this.logger.verbose(`Order ${clOrdId} was pending past its bound, but Scrypt confirms it filled`); + return true; + } + + if (cancellation === ScryptCancellation.SETTLED) { + this.pendingCancelAttempts.delete(clOrdId); + this.pendingSince.delete(clOrdId); + const ageMinutes = Util.minutesDiff(orderCreated); + throw new ScryptOrderStuckPendingError( + `Order ${clOrdId} is ${Math.round(ageMinutes)} minutes old and currently reports status ` + + `${orderInfo.status}, past the ${PENDING_STUCK_AFTER_MINUTES}-minute pending bound, and Scrypt ` + + `confirms nothing can execute under it any more`, + ); + } + + // UNCONFIRMED: nothing may be concluded. Without a confirmed cancel the reference may still be live + // in the book, so the order keeps waiting rather than being given up unconfirmed. + this.logger.warn( + `Order ${clOrdId} is pending (${orderInfo.status}) past its bound, but Scrypt would not confirm a cancel, waiting...`, + ); return false; + } } } @@ -845,8 +1097,154 @@ export class ScryptService extends PricingProvider { }; } - private async cancelOrder(clOrdId: string, from: string, to: string): Promise { + /** + * Ask the venue to make sure a reference cannot execute any more, and report what that established. + * + * For giving up on an order whose outcome was never observed. The danger there is never the order itself + * but a request still live in the book: hand the funds back to a rule while one sits open and a late fill + * spends them twice. Cancelling removes that possibility outright, which beats estimating when it has + * passed — and unlike a re-send, a cancel can never create anything. + * + * Three outcomes, because a cancel does not only ever mean "nothing happened": + * - SETTLED — terminal with nothing filled (cancelled or rejected), or the venue does not know the + * reference at all. Both mean nothing can execute under it, which is what lets the caller give the + * order up — the first outright, the second as an inference from the venue's own words rather than a + * statement about execution. See SCRYPT_UNKNOWN_ORDER for what that inference rests on. + * - EXECUTED — it reached a terminal state with something filled. Like a cancelled reference it cannot + * trade further, so the caller may give the order up; the fill has already moved the venue balance + * that the rule replans from. Reported separately from SETTLED because "something happened here" is + * worth seeing in a log and worth reconciling against. + * + * Terminal is the operative word: a refused cancel carries the order's last known state, so a + * partially filled order that could NOT be cancelled reports a fill while staying wide open. + * - UNCONFIRMED — no usable answer. Nothing may be concluded from it. + * + * Symbol resolution is the one thing that differs between this and {@link cancelIfOutstandingBySymbol}; + * everything else (send, evaluate, guard, catch) lives once in {@link cancelIfOutstandingCore} so neither + * can drift from the other. + */ + async cancelIfOutstanding(clOrdId: string, from: string, to: string): Promise { + return this.cancelIfOutstandingCore(clOrdId, async () => (await this.getTradePair(from, to)).symbol); + } + + /** + * Same outcome as {@link cancelIfOutstanding}, for a reference whose trade pair cannot be rebuilt locally — + * typically a command no longer in `ScryptAdapterCommands` (rename/removal), where a `paramMap` that + * happens to still carry a `tradeAsset` is not a guarantee the command was ever a plain sell/buy. + * + * The venue's own order-status reply already names the symbol a reference lives under + * (`ScryptOrderInfo.symbol`), straight from the venue rather than reconstructed from configuration that may + * no longer match reality (a delisted or renamed security would make `getTradePair` throw even though the + * order in question is still very much live under its original symbol). Every reference the venue can + * still show us is therefore cancellable through the symbol it hands back — there is no "symbol not + * determinable" wait path left, only the venue not knowing the reference at all (SETTLED, same inference as + * the `UnknownOrder` case below) or not answering at all (the caller's own lookup decides what to do with + * that, not this method). + */ + async cancelIfOutstandingBySymbol(clOrdId: string, symbol: string): Promise { + return this.cancelIfOutstandingCore(clOrdId, async () => symbol); + } + + private async cancelIfOutstandingCore( + clOrdId: string, + // Deferred rather than a plain string: the symbol lookup used by cancelIfOutstanding has to run INSIDE + // this method's try, exactly as it did when cancelOrder resolved it internally — otherwise a failing + // getTradePair would escape uncaught instead of settling into UNCONFIRMED like every other cancel failure. + resolveSymbol: () => Promise, + ): Promise { + try { + const symbol = await resolveSymbol(); + const report = await this.cancelOrderBySymbol(clOrdId, symbol); + + const filled = Number(report.CumQty); + + // An unreadable quantity is not a zero one. Concluding "nothing filled" from a value that could not + // be parsed is exactly how a real fill gets dropped, so it settles nothing and the caller waits. + // + // Emptiness has to be caught separately: Number('') and Number(' ') are 0, not NaN, so a missing + // quantity would otherwise pass the finite check and read as an untouched order. A negative one is + // rejected for the same reason rather than compared away: a cumulative filled size cannot be below + // zero, so a venue reporting one is not describing an untouched order — it is not being understood, + // and only the checks below would quietly treat it as though nothing had traded. + if (!report.CumQty?.trim() || !Number.isFinite(filled) || filled < 0) { + this.logger.warn(`Cancel of order ${clOrdId} reported an unreadable filled size (${report.CumQty})`); + + return ScryptCancellation.UNCONFIRMED; + } + + const refusedAsUnknown = + report.ExecType === SCRYPT_CANCEL_REJECTED && report.CxlRejReason === SCRYPT_UNKNOWN_ORDER; + + // Checked before anything else: a report claiming the venue has no record of this order while + // reporting a fill on it disagrees with itself, and that is true whatever its status says. Deciding + // on the status first would let the same contradiction through with a terminal one attached. + if (refusedAsUnknown && filled > 0) { + this.logger.warn( + `Cancel of order ${clOrdId} was refused as unknown yet reports ${report.CumQty} filled — the report contradicts itself, settling nothing`, + ); + + return ScryptCancellation.UNCONFIRMED; + } + + // Only a terminal state answers the question this method asks. A refused cancel comes back carrying + // the order's LAST KNOWN state, so a partially filled order that could not be cancelled reports a + // fill while remaining wide open — reading the fill alone would call that finished and let the + // caller walk away from a reference that can still trade. + // + // Which states are terminal is decided in one place for this venue, not restated here: a rejected + // order is just as final as a cancelled one, and a second list would be free to disagree with the + // first — leaving an order that provably cannot trade stuck for want of being recognised. + if (this.isTerminalExecutionReport(report)) { + if (filled > 0) { + this.logger.warn( + `Cancel of order ${clOrdId} came back terminal with ${report.CumQty} already filled — it executed, and the fill has to be reconciled against the venue balance`, + ); + + return ScryptCancellation.EXECUTED; + } + + return ScryptCancellation.SETTLED; + } + + // The venue does not know this reference. Taken together with the order's age and its failed status + // lookup, that is treated as settled — see SCRYPT_UNKNOWN_ORDER for what that evidence covers and + // why it is an inference rather than a guarantee. + if (refusedAsUnknown) { + this.logger.verbose(`Scrypt has no such order to cancel for ${clOrdId}`); + + return ScryptCancellation.SETTLED; + } + + this.logger.warn( + `Cancel of order ${clOrdId} left it in state ${report.OrdStatus}${ + report.CxlRejReason ? ` (${report.CxlRejReason})` : '' + } — nothing settled`, + ); + + return ScryptCancellation.UNCONFIRMED; + } catch (e) { + // No rejection branch here on purpose: this venue answers a refused cancel with an execution report, + // not an exception, and that is read above. What reaches this catch is anything that stopped the cancel + // from being answered — the symbol lookup it starts with, or the send and its wait. + // + // Not all of those got as far as writing, but this cannot tell which did, and that is the whole reason + // to treat them alike: an unconfirmed cancel may have taken effect at the venue while the cached report + // still shows the order open, and a non-terminal entry is never refreshed, so every later check would + // wait on a picture that cannot change. Dropping it costs one lookup when nothing was ever sent, and + // avoids a permanently stale one when something was. + this.forgetExecutionReport(clOrdId); + this.logger.warn(`Cancel of order ${clOrdId} went unconfirmed: ${e.message}`); + + return ScryptCancellation.UNCONFIRMED; + } + } + + private async cancelOrder(clOrdId: string, from: string, to: string): Promise { const { symbol } = await this.getTradePair(from, to); + return this.cancelOrderBySymbol(clOrdId, symbol); + } + + private async cancelOrderBySymbol(clOrdId: string, symbol: string): Promise { const newClOrdId = randomUUID(); const cancelData = { @@ -859,11 +1257,25 @@ export class ScryptService extends PricingProvider { ScryptMessageType.ORDER_CANCEL_REQUEST, [cancelData], ScryptMessageType.EXECUTION_REPORT, - (reports) => reports.find((r) => r.OrigClOrdID === clOrdId || r.ClOrdID === newClOrdId) ?? null, + // PendingCancel is the venue saying "working on it", not an answer. Taking the first report that + // merely mentions this order would freeze that intermediate state as the result — and since the + // waiter unsubscribes on its first match, the real terminal report that follows would never be seen. + (reports) => + reports.find( + (r) => + (r.OrigClOrdID === clOrdId || r.ClOrdID === newClOrdId) && r.OrdStatus !== ScryptOrderStatus.PENDING_CANCEL, + ) ?? null, 60000, ); - return report.OrdStatus === ScryptOrderStatus.CANCELED; + // Deliberately not cached under the cancelled order's own id. The venue tags a cancel confirmation with + // the CANCEL request's id, so filing it under the order would make a later status lookup read that + // order as terminally cancelled — and a cleanup cancellation says nothing about the order as a whole: + // its sibling references may still be unsettled and live. A lookup that then reports the order as known + // would take it out of quarantine and let the completion check open a replacement beside them, which is + // the double execution this path exists to prevent. + + return report; } private async editOrder( diff --git a/src/main.ts b/src/main.ts index 48e445c257..96976213a5 100644 --- a/src/main.ts +++ b/src/main.ts @@ -4,7 +4,7 @@ // internal HTTP usage is auto-instrumented. import './tracing'; import './polyfills'; // registers global EventSource for @arkade-os/sdk; see src/polyfills.ts -import { ValidationPipe, VersioningType } from '@nestjs/common'; +import { VersioningType } from '@nestjs/common'; import { NestFactory } from '@nestjs/core'; import { WsAdapter } from '@nestjs/platform-ws'; import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; @@ -20,6 +20,7 @@ import { AppModule } from './app.module'; import { Config, Environment } from './config/config'; import { ApiExceptionFilter } from './shared/filters/exception.filter'; import { apiTraceMiddleware, maskUrl } from './shared/middlewares/api-trace.middleware'; +import { DetailedValidationPipe } from './shared/pipes/detailed-validation.pipe'; import { DfxLogger } from './shared/services/dfx-logger'; import { AccountChangedWebhookDto } from './subdomains/generic/user/services/webhook/dto/account-changed-webhook.dto'; import { @@ -86,7 +87,9 @@ async function bootstrap() { defaultVersion: [Config.defaultVersion], }); app.useGlobalPipes( - new ValidationPipe({ + // Same validation and same 400 body as the stock ValidationPipe; it additionally carries the + // rejected values through to the exception filter's log line. + new DetailedValidationPipe({ whitelist: true, transformOptions: { exposeUnsetFields: false, diff --git a/src/shared/auth/__tests__/role.guard.spec.ts b/src/shared/auth/__tests__/role.guard.spec.ts index abf5a6d50a..5f4637084b 100644 --- a/src/shared/auth/__tests__/role.guard.spec.ts +++ b/src/shared/auth/__tests__/role.guard.spec.ts @@ -1,6 +1,17 @@ -import { ExecutionContext } from '@nestjs/common'; -import { hasRoleAccess, RoleGuard } from '../role.guard'; -import { UserRole } from '../user-role.enum'; +import { ExecutionContext, HttpStatus } from '@nestjs/common'; + +// The clearance Set is primed by cron; mocked here so the guard's gating logic is tested in isolation +// from the cron/DB plumbing. +jest.mock('src/shared/auth/staff-kyc-clearance', () => ({ + HasStaffKycClearance: jest.fn(), +})); + +import { StaffKycRequiredException } from 'src/shared/auth/exceptions/staff-kyc-required.exception'; +import { HasStaffKycClearance } from 'src/shared/auth/staff-kyc-clearance'; +import { hasRoleAccess, hasStaffAccess, RoleGuard, rolesSatisfying } from '../role.guard'; +import { KycGatedRoles, UserRole } from '../user-role.enum'; + +const hasStaffKycClearanceMock = HasStaffKycClearance as jest.MockedFunction; // Pins the hierarchy checks previously encoded in ad-hoc constants // (`ADMIN_ROLES.includes(role)`, `[UserRole.SUPPORT, UserRole.COMPLIANCE, ...ADMIN_ROLES].includes(role)`, @@ -71,15 +82,91 @@ describe('hasRoleAccess', () => { expect(hasRoleAccess(UserRole.ADMIN, undefined)).toBe(false); expect(hasRoleAccess(UserRole.SUPPORT, undefined)).toBe(false); }); + + // Entry roles with no `additionalRoles` entry at all (no super-roles): only the role itself matches. + it('matches only the role itself for an entry role without super-roles', () => { + expect(hasRoleAccess(UserRole.SUPER_ADMIN, UserRole.SUPER_ADMIN)).toBe(true); + expect(hasRoleAccess(UserRole.SUPER_ADMIN, UserRole.ADMIN)).toBe(false); + expect(hasRoleAccess(UserRole.CUSTODY, UserRole.ADMIN)).toBe(false); + }); +}); + +// The predicate used by the few privilege checks that live in business logic instead of a gate +// (protected KYC files, ownership-independent history access, official support replies). +describe('hasStaffAccess', () => { + afterEach(() => jest.resetAllMocks()); + + it('denies a caller whose role does not satisfy the entry role, without consulting the clearance', () => { + hasStaffKycClearanceMock.mockReturnValue(true); + + expect(hasStaffAccess(UserRole.COMPLIANCE, { role: UserRole.USER, account: 1 })).toBe(false); + expect(hasStaffKycClearanceMock).not.toHaveBeenCalled(); + }); + + it('denies an undefined jwt', () => { + expect(hasStaffAccess(UserRole.COMPLIANCE, undefined)).toBe(false); + }); + + describe.each(KycGatedRoles)('gated entry role: %s', (entryRole) => { + it('grants a cleared account', () => { + hasStaffKycClearanceMock.mockReturnValue(true); + + expect(hasStaffAccess(entryRole, { role: entryRole, account: 1 })).toBe(true); + expect(hasStaffKycClearanceMock).toHaveBeenCalledWith(1); + }); + + it('denies an uncleared account despite the correct role', () => { + hasStaffKycClearanceMock.mockReturnValue(false); + + expect(hasStaffAccess(entryRole, { role: entryRole, account: 1 })).toBe(false); + }); + }); + + it('does not require clearance for an ungated entry role', () => { + hasStaffKycClearanceMock.mockReturnValue(false); + + expect(hasStaffAccess(UserRole.USER, { role: UserRole.ADMIN, account: 1 })).toBe(true); + expect(hasStaffKycClearanceMock).not.toHaveBeenCalled(); + }); +}); + +describe('rolesSatisfying', () => { + it('covers the gated entry roles plus their super-roles', () => { + const roles = rolesSatisfying(KycGatedRoles); + + expect(roles).toEqual(expect.arrayContaining(KycGatedRoles)); + // SUPER_ADMIN satisfies every gate via the hierarchy without being listed in KycGatedRoles — + // omitting it from the clearance sync would lock super admins out of every elevated endpoint. + expect(roles).toContain(UserRole.SUPER_ADMIN); + expect(roles).not.toContain(UserRole.USER); + expect(roles).not.toContain(UserRole.ACCOUNT); + }); + + it('deduplicates roles reachable through several entry roles', () => { + expect(rolesSatisfying(KycGatedRoles)).toHaveLength(new Set(rolesSatisfying(KycGatedRoles)).size); + }); + + it('returns just the entry role when it has no super-roles', () => { + expect(rolesSatisfying([UserRole.SUPER_ADMIN])).toEqual([UserRole.SUPER_ADMIN]); + }); + + it('returns an empty list for no entry roles', () => { + expect(rolesSatisfying([])).toEqual([]); + }); }); describe('RoleGuard (multi-role access)', () => { - function contextFor(role?: UserRole): ExecutionContext { + function contextFor(role?: UserRole, account = 1): ExecutionContext { return { - switchToHttp: () => ({ getRequest: () => ({ user: role ? { role } : undefined }) }), + switchToHttp: () => ({ getRequest: () => ({ user: role ? { role, account } : undefined }) }), } as unknown as ExecutionContext; } + // Default to a cleared account so the pre-existing role-hierarchy expectations below keep testing + // the hierarchy, not the KYC gate; the gate gets its own describe block. + beforeEach(() => hasStaffKycClearanceMock.mockReturnValue(true)); + afterEach(() => jest.resetAllMocks()); + it('grants access to the single entry role and its super-roles, denies others', () => { expect(RoleGuard(UserRole.COMPLIANCE).canActivate(contextFor(UserRole.COMPLIANCE))).toBe(true); expect(RoleGuard(UserRole.COMPLIANCE).canActivate(contextFor(UserRole.ADMIN))).toBe(true); @@ -98,3 +185,94 @@ describe('RoleGuard (multi-role access)', () => { expect(RoleGuard(UserRole.COMPLIANCE, UserRole.DEBUG).canActivate(contextFor(undefined))).toBe(false); }); }); + +describe('RoleGuard (staff KYC gate on elevated endpoints)', () => { + function contextFor(role?: UserRole, account?: number): ExecutionContext { + return { + switchToHttp: () => ({ getRequest: () => ({ user: role ? { role, account } : undefined }) }), + } as unknown as ExecutionContext; + } + + afterEach(() => jest.resetAllMocks()); + + describe.each(KycGatedRoles)('entry role: %s', (entryRole) => { + it('denies the matching role when the account has no KYC clearance', () => { + hasStaffKycClearanceMock.mockReturnValue(false); + + // Throws rather than returning false, so the caller learns the reason instead of a bare 403. + expect(() => RoleGuard(entryRole).canActivate(contextFor(entryRole, 42))).toThrow(StaffKycRequiredException); + }); + + it('grants the matching role when the account is cleared', () => { + hasStaffKycClearanceMock.mockReturnValue(true); + + expect(RoleGuard(entryRole).canActivate(contextFor(entryRole, 42))).toBe(true); + expect(hasStaffKycClearanceMock).toHaveBeenCalledWith(42); + }); + }); + + it('gates super-roles too — an uncleared ADMIN loses every elevated endpoint', () => { + hasStaffKycClearanceMock.mockReturnValue(false); + + expect(() => RoleGuard(UserRole.SUPPORT).canActivate(contextFor(UserRole.ADMIN, 42))).toThrow( + StaffKycRequiredException, + ); + expect(() => + RoleGuard(UserRole.COMPLIANCE, UserRole.DEBUG).canActivate(contextFor(UserRole.SUPER_ADMIN, 42)), + ).toThrow(StaffKycRequiredException); + }); + + it('does not gate ordinary endpoints, even for a staff caller without clearance', () => { + hasStaffKycClearanceMock.mockReturnValue(false); + + // An admin using ordinary customer functionality is not touching an elevated endpoint. + expect(RoleGuard(UserRole.USER).canActivate(contextFor(UserRole.ADMIN, 42))).toBe(true); + expect(RoleGuard(UserRole.ACCOUNT).canActivate(contextFor(UserRole.ADMIN, 42))).toBe(true); + expect(hasStaffKycClearanceMock).not.toHaveBeenCalled(); + }); + + it('treats a gate that also admits an ungated entry role as an ordinary endpoint', () => { + hasStaffKycClearanceMock.mockReturnValue(false); + + // The gate is a property of the endpoint: one that is open to plain users is not elevated, + // regardless of which entry role the caller happens to come in through. + expect(RoleGuard(UserRole.ADMIN, UserRole.USER).canActivate(contextFor(UserRole.ADMIN, 42))).toBe(true); + }); + + it('denies an elevated endpoint when the token carries no account id', () => { + // Company tokens (and any future addressless token) have no `account` — fail closed rather than + // letting `undefined` slip through the clearance lookup. + hasStaffKycClearanceMock.mockImplementation((account) => account != null); + + expect(() => RoleGuard(UserRole.ADMIN).canActivate(contextFor(UserRole.ADMIN, undefined))).toThrow( + StaffKycRequiredException, + ); + }); + + // The point of throwing: a client — a person, a script, or an agent — must be able to tell this apart + // from a removed role without matching on prose. + it('answers 403 with a machine-readable code and an actionable message', () => { + hasStaffKycClearanceMock.mockReturnValue(false); + + let thrown: StaffKycRequiredException; + try { + RoleGuard(UserRole.ADMIN).canActivate(contextFor(UserRole.ADMIN, 42)); + } catch (e) { + thrown = e as StaffKycRequiredException; + } + + expect(thrown.getStatus()).toBe(HttpStatus.FORBIDDEN); + expect(thrown.getResponse()).toEqual({ + code: 'STAFF_KYC_REQUIRED', + message: expect.stringContaining('KYC level 50'), + }); + }); + + // A wrong role is a different situation with a different fix, so it must not produce the KYC answer. + it('still returns a plain false when the role itself does not satisfy the gate', () => { + hasStaffKycClearanceMock.mockReturnValue(false); + + expect(RoleGuard(UserRole.ADMIN).canActivate(contextFor(UserRole.USER, 42))).toBe(false); + expect(hasStaffKycClearanceMock).not.toHaveBeenCalled(); + }); +}); diff --git a/src/shared/auth/__tests__/staff-kyc-clearance.spec.ts b/src/shared/auth/__tests__/staff-kyc-clearance.spec.ts new file mode 100644 index 0000000000..73877a52cb --- /dev/null +++ b/src/shared/auth/__tests__/staff-kyc-clearance.spec.ts @@ -0,0 +1,46 @@ +import { HasStaffKycClearance, SetStaffKycClearance } from '../staff-kyc-clearance'; + +describe('staff KYC clearance', () => { + // reset the shared module-level Set so state does not leak between tests + afterEach(() => SetStaffKycClearance([])); + + it('clears an account id present in the primed list', () => { + SetStaffKycClearance([123, 456]); + + expect(HasStaffKycClearance(123)).toBe(true); + expect(HasStaffKycClearance(456)).toBe(true); + }); + + it('denies an account id not present in the list', () => { + SetStaffKycClearance([123]); + + expect(HasStaffKycClearance(999)).toBe(false); + }); + + it('denies an undefined account', () => { + SetStaffKycClearance([123]); + + expect(HasStaffKycClearance(undefined)).toBe(false); + }); + + // The inverse of the JWT denylist behaviour, and the point of the whole gate: no clearance data + // means no elevated access, never the other way round. + it('fails CLOSED before it is ever primed', () => { + expect(HasStaffKycClearance(123)).toBe(false); + }); + + it('fails CLOSED on an empty list', () => { + SetStaffKycClearance([]); + + expect(HasStaffKycClearance(123)).toBe(false); + }); + + it('revokes clearance as soon as an account drops out of the list', () => { + SetStaffKycClearance([123]); + expect(HasStaffKycClearance(123)).toBe(true); + + SetStaffKycClearance([]); + + expect(HasStaffKycClearance(123)).toBe(false); + }); +}); diff --git a/src/shared/auth/exceptions/staff-kyc-required.exception.ts b/src/shared/auth/exceptions/staff-kyc-required.exception.ts new file mode 100644 index 0000000000..c93d96ae5a --- /dev/null +++ b/src/shared/auth/exceptions/staff-kyc-required.exception.ts @@ -0,0 +1,15 @@ +import { ForbiddenException } from '@nestjs/common'; + +// Answer for a staff caller who holds the role but is not KYC-cleared (see `HasStaffKycClearance`). +// A bare `false` from the guard would produce the generic "Forbidden resource", which is +// indistinguishable from "your role was removed" — leaving the caller, and any tooling in front of +// it, with no way to tell that the fix is to complete an identification. The machine-readable `code` +// follows the existing `TFA_REQUIRED` pattern so clients can branch on it instead of matching text. +export class StaffKycRequiredException extends ForbiddenException { + constructor() { + super({ + code: 'STAFF_KYC_REQUIRED', + message: 'Staff access requires a completed identification: KYC level 50 and a verified name on your account', + }); + } +} diff --git a/src/shared/auth/role.guard.ts b/src/shared/auth/role.guard.ts index 16c8d011ec..cd8dfe0ce4 100644 --- a/src/shared/auth/role.guard.ts +++ b/src/shared/auth/role.guard.ts @@ -1,5 +1,7 @@ import { CanActivate, ExecutionContext } from '@nestjs/common'; -import { UserRole } from 'src/shared/auth/user-role.enum'; +import { StaffKycRequiredException } from 'src/shared/auth/exceptions/staff-kyc-required.exception'; +import { HasStaffKycClearance } from 'src/shared/auth/staff-kyc-clearance'; +import { KycGatedRoles, UserRole } from 'src/shared/auth/user-role.enum'; // Role hierarchy: `additionalRoles[entryRole]` are the roles that also satisfy an // `entryRole` requirement (super-roles). Single source of truth for role checks — @@ -52,6 +54,32 @@ export function hasRoleAccess(entryRole: UserRole, userRole: UserRole | undefine return entryRole === userRole || (additionalRoles[entryRole]?.includes(userRole) ?? false); } +/** + * `hasRoleAccess` plus the staff KYC clearance that `RoleGuard` applies to elevated endpoints — for + * the handful of places that decide staff privileges in business logic rather than through a gate. + * Those sit on endpoints that are NOT role-gated (OptionalJwtAuthGuard / an ACCOUNT gate), so without + * this the KYC requirement would be bypassable: an uncleared admin would lose `/admin` yet keep + * protected KYC file downloads and ownership-independent access to any customer's history. + * + * Not needed where a privilege check merely refines behaviour behind an already-gated endpoint (e.g. + * the `isAdmin` distinctions in the support note services) — the gate has run by then. + */ +export function hasStaffAccess(entryRole: UserRole, jwt: { role?: UserRole; account?: number } | undefined): boolean { + if (!hasRoleAccess(entryRole, jwt?.role)) return false; + if (!KycGatedRoles.includes(entryRole)) return true; + return HasStaffKycClearance(jwt?.account); +} + +/** + * Every role that satisfies at least one of `entryRoles` — the entry roles themselves plus their + * super-roles per `additionalRoles`. Derived from the same map as `hasRoleAccess` so that a hierarchy + * change cannot desync the two: `StaffKycClearanceService` uses this to decide whose KYC clearance + * to track, and would otherwise silently lock out a role newly granted access to a gated endpoint. + */ +export function rolesSatisfying(entryRoles: UserRole[]): UserRole[] { + return [...new Set(entryRoles.flatMap((entryRole) => [entryRole, ...(additionalRoles[entryRole] ?? [])]))]; +} + class RoleGuardClass implements CanActivate { private readonly entryRoles: UserRole[]; @@ -60,8 +88,27 @@ class RoleGuardClass implements CanActivate { } canActivate(context: ExecutionContext): boolean { - const userRole = context.switchToHttp().getRequest().user?.role; - return this.entryRoles.some((entryRole) => hasRoleAccess(entryRole, userRole)); + const user = context.switchToHttp().getRequest().user; + if (!this.entryRoles.some((entryRole) => hasRoleAccess(entryRole, user?.role))) return false; + + // Elevated endpoint: an identified natural person must be behind the account (see KycGatedRoles). + // The gate is a property of the ENDPOINT, not of the caller, so it applies only when EVERY entry + // role is gated — a gate that also admits e.g. UserRole.USER is an ordinary endpoint that an admin + // happens to reach through the role hierarchy, and must not start demanding staff KYC. + // + // Throws rather than returning false: a bare false becomes the generic "Forbidden resource", which + // a caller cannot tell apart from a removed role, so neither staff nor tooling would learn that the + // fix is to complete an identification. `JwtUserActiveGuard` calls this guard programmatically, but + // only with UserRole.USER, which is never elevated — so it still gets a boolean. + if (this.isElevated && !HasStaffKycClearance(user?.account)) throw new StaffKycRequiredException(); + + return true; + } + + // No empty-list guard needed: `canActivate` has already returned false by then, since no role can + // satisfy an empty entry-role list. + private get isElevated(): boolean { + return this.entryRoles.every((entryRole) => KycGatedRoles.includes(entryRole)); } } diff --git a/src/shared/auth/staff-kyc-clearance.ts b/src/shared/auth/staff-kyc-clearance.ts new file mode 100644 index 0000000000..69275ff88c --- /dev/null +++ b/src/shared/auth/staff-kyc-clearance.ts @@ -0,0 +1,26 @@ +// Staff KYC clearance ALLOWlist — the inverse of the JWT denylists in ProcessService. Elevated +// endpoints (every `RoleGuard` whose entry roles are all in `KycGatedRoles`) require, on top of the +// role, that an identified natural person is behind the calling account: `kycLevel >= LEVEL_50` AND a +// non-empty `verifiedName`. `StaffKycClearanceService` derives the cleared account (user data) ids +// from the DB into the `staffKycClearance` setting; `ProcessService` primes this Set from it, so +// revoking a staff member's KYC takes effect on live tokens within one refresh interval — no +// re-login, no JWT-secret rotation. +// +// Fail-CLOSED, unlike the denylists: a not-yet-primed or empty Set denies every elevated endpoint. +// That asymmetry is deliberate — a DB or cron outage must never silently re-open admin access — and +// it is why ProcessService awaits the first prime before HTTP starts, and why a failing resync keeps +// the last known Set rather than clearing it. +// +// Deliberately its own module rather than living next to the denylists in `process.service.ts`: +// RoleGuard is imported by nearly every controller, and importing ProcessService from it would pull +// `config.ts` (and with it the whole blockchain/node-pty dependency chain) into the auth path. +let StaffKycClearedAccounts: Set = new Set(); + +export function HasStaffKycClearance(account: number | undefined): boolean { + return account != null && StaffKycClearedAccounts.has(account); +} + +// Only ProcessService should call this — the cron owns the lifecycle of the Set. +export function SetStaffKycClearance(accounts: number[]): void { + StaffKycClearedAccounts = new Set(accounts); +} diff --git a/src/shared/auth/user-role.enum.ts b/src/shared/auth/user-role.enum.ts index d1c8bc2e48..3fb67687a2 100644 --- a/src/shared/auth/user-role.enum.ts +++ b/src/shared/auth/user-role.enum.ts @@ -25,3 +25,13 @@ export enum UserRole { // must pass an independent TOTP second factor (never a mail code to the same inbox as the magic link). // Priority-ordered (highest privilege first) for mail-login role resolution. export const StaffRoles = [UserRole.COMPLIANCE, UserRole.SUPPORT, UserRole.REALUNIT]; + +// Entry roles that mark an endpoint as elevated: reaching it requires an identified natural person +// behind the account, on top of the role itself. `RoleGuard` therefore demands staff KYC clearance +// (`kycLevel >= LEVEL_50` AND a non-empty `verifiedName`, see `HasStaffKycClearance`) whenever every +// entry role of a gate is listed here. Distinct from `StaffRoles` above, which is about mail-login +// role resolution — this list is about endpoint sensitivity and also covers ADMIN and DEBUG. +// +// Not listed, deliberately: BANKING_BOT and CUSTODY are non-staff entry roles and stay ungated, so a +// cleared-role holder reaching those endpoints via the `additionalRoles` hierarchy is not KYC-gated. +export const KycGatedRoles = [UserRole.ADMIN, UserRole.DEBUG, UserRole.COMPLIANCE, UserRole.SUPPORT, UserRole.REALUNIT]; diff --git a/src/shared/decorators/__tests__/log-rejected-value.decorator.spec.ts b/src/shared/decorators/__tests__/log-rejected-value.decorator.spec.ts new file mode 100644 index 0000000000..3bc26c70fc --- /dev/null +++ b/src/shared/decorators/__tests__/log-rejected-value.decorator.spec.ts @@ -0,0 +1,77 @@ +import { LogRejectedValue, loggableRejectedValues } from 'src/shared/decorators/log-rejected-value.decorator'; + +enum Mode { + FAST = 'Fast', + SLOW = 'Slow', +} + +enum Speed { + LOW = 0, + HIGH = 1, +} + +class Declaring { + @LogRejectedValue(Mode) + mode: string; + + @LogRejectedValue(['Bank', 'Crypto']) + method: string; + + free: string; +} + +class Silent { + mode: string; +} + +class Numeric { + @LogRejectedValue(Speed) + speed: number; +} + +class Extending extends Declaring { + @LogRejectedValue([1, 2, true]) + own: string; +} + +describe('LogRejectedValue', () => { + it('takes the values of an enum object', () => { + expect([...(loggableRejectedValues(Declaring, 'mode') ?? [])]).toEqual([ + ['fast', 'Fast'], + ['slow', 'Slow'], + ]); + }); + + it('takes a plain list, including numbers and booleans', () => { + expect([...(loggableRejectedValues(Extending, 'own') ?? [])]).toEqual([ + ['1', '1'], + ['2', '2'], + ['true', 'true'], + ]); + }); + + it('leaves the reverse mapping of a numeric enum out', () => { + // `{ LOW: 0, HIGH: 1 }` reads back as `['LOW', 'HIGH', 0, 1]`: the member names are not values + // the field accepts, and taking them would match a request that sent one of them. + expect([...(loggableRejectedValues(Numeric, 'speed') ?? [])]).toEqual([ + ['0', '0'], + ['1', '1'], + ]); + }); + + it('reports nothing for a property that declared nothing', () => { + expect(loggableRejectedValues(Declaring, 'free')).toBeUndefined(); + expect(loggableRejectedValues(Silent, 'mode')).toBeUndefined(); + }); + + it('reports nothing for anything that is not a class', () => { + expect(loggableRejectedValues(undefined, 'mode')).toBeUndefined(); + expect(loggableRejectedValues({ mode: 1 }, 'mode')).toBeUndefined(); + expect(loggableRejectedValues('Declaring', 'mode')).toBeUndefined(); + }); + + it('inherits what a parent declared without a subclass reaching back into it', () => { + expect(loggableRejectedValues(Extending, 'mode')?.get('fast')).toBe('Fast'); + expect(loggableRejectedValues(Declaring, 'own')).toBeUndefined(); + }); +}); diff --git a/src/shared/decorators/log-rejected-value.decorator.ts b/src/shared/decorators/log-rejected-value.decorator.ts new file mode 100644 index 0000000000..f78dac7a53 --- /dev/null +++ b/src/shared/decorators/log-rejected-value.decorator.ts @@ -0,0 +1,71 @@ +// Per class, the values each property may contribute to a log line, keyed by property and held on +// the class itself. Shared between the decorator and its reader so the two can never drift. +const LOG_REJECTED_VALUE = Symbol('logRejectedValue'); + +type Loggable = { [LOG_REJECTED_VALUE]?: Map> }; + +/** The values a field may render, as an enum object or a plain list. */ +export type LoggableValues = Record | readonly (string | number | boolean)[]; + +/** + * Declares which values of a DTO property may be written to a log line when the property is + * rejected. + * + * A rejection names the field and the values it accepts, never the one that came - so a client + * sending a wrong constant produces the same line as one sending nothing, and the fix, usually one + * constant in one client, cannot be named from the logs. Naming it needs the value; logging the + * value as it arrived does not work, because a constraint bounds what is accepted and a rejected + * value is by definition outside it. A field constrained to three payment methods rejects an + * account number exactly as readily as it rejects a typo. + * + * So the field declares a set instead, and only a value in that set is rendered - matched without + * regard to case, and rendered from the set rather than from the request, so what reaches the log + * is a constant of this program either way. Everything else keeps its shape and loses its content. + * + * The set to pass is the one a wrong value plausibly comes from and the field does not accept: for + * a field taking the fiat payment methods, the full payment-method union, whose crypto members are + * exactly what a client sends there by mistake. + */ +export function LogRejectedValue(values: LoggableValues): PropertyDecorator { + const loggable = new Map(declared(values).map(canonical)); + + return (target: object, property: string | symbol) => { + const type = target.constructor as Loggable; + + // A subclass starts from what it inherits and grows its own map, so declaring a property on it + // never reaches back into the class it extends. + const declared = Object.prototype.hasOwnProperty.call(type, LOG_REJECTED_VALUE) + ? (type[LOG_REJECTED_VALUE] as Map>) + : new Map(type[LOG_REJECTED_VALUE] ?? []); + + declared.set(property, loggable); + Object.defineProperty(type, LOG_REJECTED_VALUE, { value: declared, configurable: true }); + }; +} + +/** + * What the given property of the given class may render, or undefined if it declared nothing. The + * keys are lower-cased; the values are the constants as they are written in the code. + */ +export function loggableRejectedValues( + type: unknown, + property: string | symbol, +): ReadonlyMap | undefined { + if (typeof type !== 'function') return undefined; + + return (type as Loggable)[LOG_REJECTED_VALUE]?.get(property); +} + +// A numeric enum object carries its reverse mapping as well, so its own member names appear among +// its values - `{ FAST: 0, SLOW: 1 }` reads back as `['FAST', 'SLOW', 0, 1]`. A name that maps to a +// number is one of those and is not a value the field ever accepts. +function declared(values: LoggableValues): (string | number | boolean)[] { + if (Array.isArray(values)) return [...values]; + + const members = values as Record; + return Object.values(members).filter((value) => typeof members[`${value}`] !== 'number'); +} + +function canonical(value: string | number | boolean): [string, string] { + return [`${value}`.toLowerCase(), `${value}`]; +} diff --git a/src/shared/filters/__tests__/exception.filter.spec.ts b/src/shared/filters/__tests__/exception.filter.spec.ts index 1f08fce476..1fd196eadc 100644 --- a/src/shared/filters/__tests__/exception.filter.spec.ts +++ b/src/shared/filters/__tests__/exception.filter.spec.ts @@ -8,7 +8,10 @@ import { NotFoundException, UnauthorizedException, } from '@nestjs/common'; +import { ValidationError } from 'class-validator'; +import { LogRejectedValue } from 'src/shared/decorators/log-rejected-value.decorator'; import { ApiExceptionFilter } from 'src/shared/filters/exception.filter'; +import { ValidationFailedException } from 'src/shared/pipes/detailed-validation.pipe'; describe('ApiExceptionFilter', () => { let filter: ApiExceptionFilter; @@ -97,6 +100,139 @@ describe('ApiExceptionFilter', () => { expect(msg).not.toContain('foo@bar.com'); }); + it('keeps the reason on one line, so a value interpolated into it cannot forge a second', () => { + // Exception messages interpolate request values (`Invalid address for ...: ${address}`), so a + // line break in one of those would otherwise reach the log as a line of its own. + filter.catch( + new BadRequestException('Invalid address for to: abc\nWARN [ApiExceptionFilter] forged line'), + host(req(), { status }), + ); + + const msg = warn.mock.calls[0][0] as string; + expect(msg).not.toContain('\n'); + expect(msg).toContain('abcWARN'); + }); + + it('caps a reason as large as the body it came from, and still masks up to the cap', () => { + // Exception messages interpolate request values, and a request body is large; the reason is + // cut to the cap, and a pattern that starts inside it is masked even though it runs past it. + const message = `${'a'.repeat(480)}someone@example.com${'b'.repeat(200_000)}`; + filter.catch(new BadRequestException(message), host(req(), { status })); + + const msg = warn.mock.calls[0][0] as string; + expect(msg).not.toContain('someone@example.com'); + expect(msg).toContain('***'); + expect(msg.length).toBeLessThan(700); + }); + + it('masks a pattern that a control character sits inside', () => { + // Removing the control character rather than replacing it puts the pattern back together, so + // the masking that runs after it sees the value as the one it is. + filter.catch( + new BadRequestException('Invalid recipient victim\u0001@example.com and victim\u0085@example.com'), + host(req(), { status }), + ); + + const msg = warn.mock.calls[0][0] as string; + expect(msg).not.toContain('victim'); + expect(msg).not.toContain('example.com'); + }); + + it('sends the response even when the body cannot be read', () => { + const unreadable = new BadRequestException('x'); + jest.spyOn(unreadable, 'getResponse').mockImplementation(() => { + throw new Error('nope'); + }); + + expect(() => filter.catch(unreadable, host(req(), { status }))).not.toThrow(); + expect(status).toHaveBeenCalledWith(400); + expect(json).toHaveBeenCalledWith({ statusCode: 400, message: 'x' }); + }); + + it('keeps sending the response when the message cannot be read', () => { + const unreadable = new BadRequestException({ + statusCode: 400, + message: [ + { + toString: () => { + throw new Error('nope'); + }, + }, + ], + }); + + expect(() => filter.catch(unreadable, host(req(), { status }))).not.toThrow(); + expect(status).toHaveBeenCalledWith(400); + expect(json).toHaveBeenCalled(); + }); + + it('keeps a failure to write the line away from a caller that already has its answer', () => { + warn.mockImplementation(() => { + throw new Error('logger down'); + }); + + expect(() => filter.catch(new BadRequestException('bad'), host(req(), { status }))).not.toThrow(); + expect(status).toHaveBeenCalledWith(400); + expect(json).toHaveBeenCalled(); + }); + + it('describes the response it is sending, not the one the exception named', () => { + // The status was replaced, so the body the exception carries no longer says what is being sent. + const mismatched = new BadRequestException('x'); + jest.spyOn(mismatched, 'getStatus').mockReturnValue(600); + + filter.catch(mismatched, host(req(), { status })); + + expect(status).toHaveBeenCalledWith(500); + expect(json).toHaveBeenCalledWith({ statusCode: 500, message: 'x' }); + }); + + it('sends the response even when the message cannot be read either', () => { + const mute = new BadRequestException('x'); + jest.spyOn(mute, 'getResponse').mockImplementation(() => { + throw new Error('nope'); + }); + Object.defineProperty(mute, 'message', { + get: () => { + throw new Error('nope'); + }, + }); + + expect(() => filter.catch(mute, host(req(), { status }))).not.toThrow(); + expect(json).toHaveBeenCalledWith({ statusCode: 400, message: 'BAD_REQUEST' }); + }); + + it('sends a server error when the status cannot be read or is not one Express sends', () => { + const broken = new BadRequestException('x'); + jest.spyOn(broken, 'getStatus').mockImplementation(() => { + throw new Error('nope'); + }); + filter.catch(broken, host(req(), { status })); + expect(status).toHaveBeenCalledWith(500); + + for (const invalid of [0, -1, NaN, 100, 199, 600, 1.5]) { + const outOfRange = new BadRequestException('x'); + jest.spyOn(outOfRange, 'getStatus').mockReturnValue(invalid); + filter.catch(outOfRange, host(req(), { status })); + expect(status).toHaveBeenLastCalledWith(500); + } + }); + + it('sends the response even when the request cannot be read', () => { + const brokenHost = { + switchToHttp: () => ({ + getResponse: () => ({ status }), + getRequest: () => { + throw new Error('nope'); + }, + }), + } as unknown as ArgumentsHost; + + expect(() => filter.catch(new BadRequestException('bad'), brokenHost)).not.toThrow(); + expect(status).toHaveBeenCalledWith(400); + expect(json).toHaveBeenCalled(); + }); + it('does NOT log routine client errors (401/403/404/429) — they are already in the access log', () => { const routine = [ new UnauthorizedException(), @@ -120,6 +256,49 @@ describe('ApiExceptionFilter', () => { expect(status).toHaveBeenCalledWith(500); }); + it('names the caller, so a rejection on an unauthenticated endpoint is attributable', () => { + filter.catch( + new BadRequestException('bad'), + host(req({ headers: { 'x-client': 'dfx-services', origin: 'https://app.dfx.swiss' } }), { status }), + ); + + const msg = warn.mock.calls[0][0] as string; + expect(msg).toContain('client=dfx-services'); + expect(msg).toContain('origin=https://app.dfx.swiss'); + }); + + it('marks a caller that identifies itself with nothing', () => { + filter.catch(new BadRequestException('bad'), host(req({ headers: {} }), { status })); + + expect(warn.mock.calls[0][0]).toContain('client=(none)'); + }); + + it('appends the rejected values of a failed validation — the message alone names only the field', () => { + class PaymentDto { + @LogRejectedValue(['Bank', 'Instant', 'Card', 'Crypto']) + paymentMethod: string; + } + + const error: ValidationError = { + property: 'paymentMethod', + value: 'Crypto', + constraints: { isEnum: 'paymentMethod must be one of the following values: Bank, Instant, Card' }, + children: [], + target: new PaymentDto(), + }; + + filter.catch( + new ValidationFailedException({ statusCode: 400, message: [error.constraints.isEnum] }, [error]), + host(req({ headers: {} }), { status }), + ); + + const msg = warn.mock.calls[0][0] as string; + expect(msg).toContain('must be one of the following values'); + expect(msg).toContain("received: paymentMethod='Crypto'"); + // the client still gets exactly the body the validation pipe built + expect(json).toHaveBeenCalledWith({ statusCode: 400, message: [error.constraints.isEnum] }); + }); + it('treats a non-HttpException as a 500 and returns a generic body', () => { filter.catch(new Error('unexpected'), host(req(), { status })); diff --git a/src/shared/filters/exception.filter.ts b/src/shared/filters/exception.filter.ts index b247e5fd19..5a256de564 100644 --- a/src/shared/filters/exception.filter.ts +++ b/src/shared/filters/exception.filter.ts @@ -1,7 +1,9 @@ import { ArgumentsHost, Catch, ExceptionFilter, HttpException, HttpStatus } from '@nestjs/common'; import { Request } from 'express'; -import { maskUrl, maskValue } from 'src/shared/middlewares/api-trace.middleware'; +import { capCharacters, maskLogText, maskUrl } from 'src/shared/middlewares/api-trace.middleware'; +import { ValidationFailedException, describeRejectedValues } from 'src/shared/pipes/detailed-validation.pipe'; import { DfxLogger } from 'src/shared/services/dfx-logger'; +import { describeCaller } from 'src/shared/utils/request-caller'; @Catch() export class ApiExceptionFilter implements ExceptionFilter { @@ -20,9 +22,28 @@ export class ApiExceptionFilter implements ExceptionFilter { catch(exception: Error, host: ArgumentsHost) { const ctx = host.switchToHttp(); const response = ctx.getResponse(); - const request = ctx.getRequest(); - const status = exception instanceof HttpException ? exception.getStatus() : HttpStatus.INTERNAL_SERVER_ERROR; + const status = ApiExceptionFilter.statusOf(exception); + // The response goes out first, and nothing it does not need is read before it. Everything the + // line renders comes from the request or from the thrower, and reading either can throw - which + // used to leave the caller with no response at all rather than with a line missing a detail. + try { + response.status(status).json(this.responseBody(exception, status)); + } catch (e) { + this.logger.error(`Failed to set error response content:`, e); + } + + // The response is out; what follows only describes it. A failure to do that must not travel back + // to a caller who already has an answer, so it ends here - including a failure of the logger, + // which is the one thing that could not be used to report it anyway. + try { + this.describe(exception, ctx.getRequest(), status); + } catch { + return; + } + } + + private describe(exception: Error, request: Request, status: number): void { const target = `${request.method} request to '${maskUrl(request.originalUrl ?? request.url ?? '')}'`; if (status >= 500) { // log server errors with the full error + stack @@ -32,39 +53,79 @@ export class ApiExceptionFilter implements ExceptionFilter { // that surfaces as a 4xx (a valid request the server wrongly rejects) is // visible in the logs instead of leaving only a bare morgan status line with // no reason (the #4105 support-ticket outage was silent for exactly this). - // The reason is masked to the same PII standard as the rest of the logs and - // length-capped, because it can embed user-supplied values. - const reason = maskValue(this.getReason(exception)).slice(0, ApiExceptionFilter.REASON_MAX_LENGTH); - this.logger.warn(`${status} on ${target}: ${reason}`); + // + // The caller markers and the rejected values are what make a steady stream of rejections + // actionable: the constraint message names the field and the allowed values, so without the + // value that arrived and a hint at who sent it, a wrong constant in a client can only be + // guessed at. + // + // All three are untrusted input and rendered as such - single-line, masked and capped. The + // reason included: an exception message can interpolate a value the request supplied. + const reason = capCharacters(maskLogText(this.getReason(exception)), ApiExceptionFilter.REASON_MAX_LENGTH); + const rejected = + exception instanceof ValidationFailedException + ? ` (received: ${describeRejectedValues(exception.validationErrors)})` + : ''; + this.logger.warn(`${status} on ${target} from ${describeCaller(request)}: ${reason}${rejected}`); } + } + // The status an HttpException carries is whatever the thrower put there: reading it can throw, and + // what comes back is not necessarily a final response at all - a 1xx is an interim one, and + // nothing outside the range it can send leaves a reading other than server error. + private static statusOf(exception: Error): number { try { - response.status(status).json( - exception instanceof HttpException - ? exception.getResponse() - : { - statusCode: status, - message: exception.message, - }, - ); - } catch (e) { - this.logger.error(`Failed to set error response content:`, e); + if (!(exception instanceof HttpException)) return HttpStatus.INTERNAL_SERVER_ERROR; + + const status = exception.getStatus(); + return Number.isInteger(status) && status >= 200 && status <= 599 ? status : HttpStatus.INTERNAL_SERVER_ERROR; + } catch { + return HttpStatus.INTERNAL_SERVER_ERROR; + } + } + + // The body an HttpException carries is whatever the thrower put there, and reading it can throw. + // A caller gets the generic body then rather than none - and also when the status it was going to + // be sent with is not the one it names, which is what a replaced status leaves behind. + private responseBody(exception: Error, status: number): unknown { + try { + if (exception instanceof HttpException && exception.getStatus() === status) return exception.getResponse(); + } catch { + // an exception that cannot say what it is gets described by what is being sent + } + + return { statusCode: status, message: ApiExceptionFilter.messageOf(exception, status) }; + } + + private static messageOf(exception: Error, status: number): string { + try { + return exception.message || (HttpStatus[status] as string); + } catch { + return HttpStatus[status] as string; } } // Human-readable rejection reason. For HttpExceptions the useful text is in the // response body (a plain message, or the class-validator error array), which is // more specific than the generic exception.message. + // + // The body is whatever the thrower put there, so reading it can throw: an array element that + // cannot be turned into a string takes `join` with it. The response has already gone out by then; + // the line loses its reason instead. private getReason(exception: Error): string { - if (exception instanceof HttpException) { - const res = exception.getResponse(); - if (typeof res === 'string') return res; + try { + if (exception instanceof HttpException) { + const res = exception.getResponse(); + if (typeof res === 'string') return res; - const message = (res as { message?: unknown }).message; - if (Array.isArray(message)) return message.join('; '); - if (typeof message === 'string') return message; - } + const message = (res as { message?: unknown }).message; + if (Array.isArray(message)) return message.join('; '); + if (typeof message === 'string') return message; + } - return exception.message; + return exception.message; + } catch { + return '(unreadable reason)'; + } } } diff --git a/src/shared/middlewares/__tests__/api-trace.middleware.spec.ts b/src/shared/middlewares/__tests__/api-trace.middleware.spec.ts index 68f7106be5..69aedcf068 100644 --- a/src/shared/middlewares/__tests__/api-trace.middleware.spec.ts +++ b/src/shared/middlewares/__tests__/api-trace.middleware.spec.ts @@ -1,4 +1,4 @@ -import { apiTraceMiddleware } from 'src/shared/middlewares/api-trace.middleware'; +import { apiTraceMiddleware, maskLogValue } from 'src/shared/middlewares/api-trace.middleware'; import { DfxLogger } from 'src/shared/services/dfx-logger'; type Emit = (res: any) => void; @@ -172,7 +172,7 @@ describe('apiTraceMiddleware', () => { expect(line.length).toBeLessThan(3 * 4500); // each of the 3 sections stays within MAX_PART // the reported serialized size proves the walk stopped at the budget // instead of stringifying the whole ~20 MB body - const [, serializedSize] = line.match(/req\.body=.*…\((\d+) chars\)/) ?? []; + const [, serializedSize] = line.match(/req\.body=.*…\((\d+) code units\)/) ?? []; expect(Number(serializedSize)).toBeLessThan(10_000); }); @@ -185,6 +185,64 @@ describe('apiTraceMiddleware', () => { expect(line).toContain(''); }); + it('cuts an oversized section between characters, so the section carries no stray surrogate', () => { + // Every value stays under MAX_STRING, so the section is cut by its own cap rather than the + // per-string one; the padding puts an astral character across that cut. + const notes = Array.from({ length: 9 }, () => 'b'.repeat(400)); + const withPadding = (padding: number): { notes: string[] } => ({ + notes: [...notes, `${'c'.repeat(padding)}${'\u{1F600}'.repeat(10)}`], + }); + let padding = 0; + while (JSON.stringify(withPadding(padding)).indexOf('\u{1F600}') < 4000 - 1) padding++; + + const { lines } = runTrace(realunitReq(withPadding(padding)), 200, (res) => res.json({})); + const line = lines.join('\n'); + + expect(line).toContain('code units)'); + expect(line).not.toContain('\ufffd'); + // no lone surrogate: a well-formed pair is one character with a code point above the range + const isLoneSurrogate = (character: string): boolean => { + const codePoint = character.codePointAt(0) as number; + return codePoint >= 0xd800 && codePoint <= 0xdfff; + }; + expect([...line].some(isLoneSurrogate)).toBe(false); + }); + + it('keeps the request target on one line as well', () => { + const req = { + method: 'GET', + originalUrl: '/v1/realunit/account/x\u0085INFO [RealUnitTrace] forged line', + headers: {}, + body: undefined, + }; + const { lines } = runTrace(req, 404, (res) => res.send('Not Found')); + + expect(lines).toHaveLength(1); + expect(lines[0]).not.toContain('\u0085'); + expect(lines[0]).toContain('/v1/realunit/account/xINFO'); + }); + + it('renders the client header like any other value from the caller', () => { + const req = realunitReq({ amount: 1 }); + req.headers['x-client'] = 'realunit-app\u0085INFO [RealUnitTrace] forged line'; + const { lines } = runTrace(req, 200, (res) => res.json({})); + + expect(lines).toHaveLength(1); + expect(lines[0]).not.toContain('\u0085'); + expect(lines[0]).toContain('client=realunit-appINFO'); + }); + + it('keeps the trace on one line, including the separators JSON.stringify leaves raw', () => { + // `JSON.stringify` escapes the control characters, but not U+2028 / U+2029. + const note = 'first\u2028second\u2029third\nfourth'; + const { lines } = runTrace(realunitReq({ note }), 200, (res) => res.json({})); + + expect(lines).toHaveLength(1); + expect(lines[0]).not.toContain('\u2028'); + expect(lines[0]).not.toContain('\u2029'); + expect(lines[0]).toContain('firstsecondthird'); + }); + it('logs metadata-only for a realunit-app call to a non-realunit path', () => { const req = { method: 'POST', @@ -207,3 +265,63 @@ describe('apiTraceMiddleware', () => { expect(nextCalled).toBe(true); }); }); + +describe('maskLogValue', () => { + it('masks personal data in the value', () => { + expect(maskLogValue('write to foo@bar.com', 100)).toBe('write to ***'); + expect(maskLogValue('from 10.0.0.1', 100)).toBe('from ***'); + }); + + it('removes everything that could break the line — newline, ANSI escape, Unicode separators', () => { + expect(maskLogValue('a\nb', 100)).toBe('ab'); + expect(maskLogValue('a\r\nb', 100)).toBe('ab'); + expect(maskLogValue('a\u001b[31mb', 100)).toBe('a[31mb'); + expect(maskLogValue('a\u2028b\u2029c', 100)).toBe('abc'); + }); + + it('caps the value and marks the cut', () => { + expect(maskLogValue('x'.repeat(10), 4)).toBe('xxxx\u2026'); + expect(maskLogValue('x'.repeat(4), 4)).toBe('xxxx'); + }); + + it('cuts between characters, so a surrogate pair is not halved at the boundary', () => { + const cut = maskLogValue(`${'a'.repeat(63)}\u{1F600}x`, 64); + + expect(cut).toBe(`${'a'.repeat(63)}\u{1F600}\u2026`); + expect(cut).not.toContain('\uFFFD'); + expect([...cut].every((character) => character.codePointAt(0) !== 0xd83d)).toBe(true); + }); + + it('reports an oversized value by length instead of masking it', () => { + // The caller's cap alone would still pay for masking the whole string first. + expect(maskLogValue('x'.repeat(513), 96)).toBe('<513 code units>'); + expect(maskLogValue('x'.repeat(512), 96)).toContain('\u2026'); + }); + + it('names the unit of the reported length, which counts code units and not characters', () => { + // 257 astral characters are 514 code units: the value the guard compares and the one reported. + expect(maskLogValue('\u{1F600}'.repeat(257), 96)).toBe('<514 code units>'); + }); + + it('masks a pattern a control character was placed next to, which removing it would join', () => { + // Removing the character puts what followed it against the end of the pattern, and the address + // no longer ends on a word boundary - so the masking also runs before the removal. + expect(maskLogValue('192.0.2.123\u0000a', 96)).toBe('***a'); + expect(maskLogValue(`0x${'a'.repeat(40)}\u0000b`, 96)).toBe('0x…b'); + }); + + it('masks a pattern that a control character was placed inside', () => { + // Removing the character rather than replacing it puts the pattern back together, so the + // masking sees it as the value it is. + expect(maskLogValue('0x1234567890abcdef1234\u0000567890abcdef12345678', 96)).toBe('0x…'); + expect(maskLogValue('192.0\u2028.2.123', 96)).toBe('***'); + expect(maskLogValue('victim\u0001@example.com', 96)).toBe('***'); + }); + + it('masks before cutting, so a truncated email cannot slip through', () => { + const masked = maskLogValue('someone@example.com', 12); + + expect(masked).not.toContain('someone@'); + expect(masked).toBe('***'); + }); +}); diff --git a/src/shared/middlewares/api-trace.middleware.ts b/src/shared/middlewares/api-trace.middleware.ts index d060d7f538..9abee57c6e 100644 --- a/src/shared/middlewares/api-trace.middleware.ts +++ b/src/shared/middlewares/api-trace.middleware.ts @@ -23,7 +23,8 @@ const WALLET_ADDRESS = /0x[0-9a-f]{40}(?![0-9a-f])/gi; const EMAIL = /[^\s"@/]{1,64}@[^\s"@/]{1,255}\.[^\s"@/.]{1,24}/g; const IPV4 = /\b\d{1,3}(?:\.\d{1,3}){3}\b/g; -const MAX_STRING = 512; // per string leaf +export const MAX_STRING = 512; // per logged string: beyond this only its length is reported +const MAX_CLIENT = 32; // per trace line: the client header is a name, not a payload const MAX_PART = 4000; // per serialized section (headers / req body / res body) const REDACT_BUDGET = 2 * MAX_PART; // per section: bounds the compute, not just the output const REDACTED = '***'; @@ -34,7 +35,85 @@ export function maskValue(s: string): string { } export function maskUrl(url: string): string { - return maskValue(url.split('?')[0]); + // The request target is client-supplied and reaches a log line: what is left of it after the + // query is dropped is rendered like any other value from the request. + return maskLogText(url.split('?')[0]); +} + +// Everything that can break a line or move a cursor in a log viewer: the control characters +// (which include the ordinary line breaks and the ANSI escape) plus the two Unicode separators +// that sit outside that category. +const LINE_BREAKING = /[\p{C}\u2028\u2029]/gu; + +/** + * Removes everything that could break a log line, so a crafted value cannot forge a second line or + * smuggle an ANSI escape into the console stream. + */ +export function singleLine(value: string): string { + return value.replace(LINE_BREAKING, ''); +} + +/** + * Renders free-form text for a log line: masked, on one line, masked again. + * + * Both passes are needed, because a character that breaks a line also breaks a pattern in either + * direction. Put inside one, it hides the pattern from a pass that runs before the removal + * (`victim\u0001@example.com`). Removing it joins what stood on either side, which can hide a + * pattern that was whole from a pass that runs after (`192.0.2.123\u0000a` becomes `192.0.2.123a`, + * where the address no longer ends on a word boundary). Neither order sees both, so both run - and + * the second pass cannot invent a match, since what the first one leaves behind is `***` and `0x…`. + */ +export function maskLogText(value: string): string { + return maskValue(singleLine(maskValue(value))); +} + +/** + * Caps a rendered value, cutting between characters rather than between code units: `slice` would + * halve a surrogate pair sitting on the boundary and leave the stray half in front of the ellipsis, + * which reaches the log as a replacement character. + * + * The walk stops at the cap rather than materializing the value first, so the work is the cap and + * not the length of what was passed - a caller holding a request-sized string (an exception message + * that interpolated a body value) would otherwise pay for all of it to render 500 characters. + */ +export function capCharacters(value: string, maxLength: number): string { + let end = 0; + for (let taken = 0; taken < maxLength; taken++) { + if (end >= value.length) return value; + end += (value.codePointAt(end) as number) > 0xffff ? 2 : 1; + } + + return end >= value.length ? value : `${value.slice(0, end)}\u2026`; +} + +/** + * Cuts to a budget in code units, moving off a surrogate pair rather than through it. That is the + * measure a section is budgeted in - `capCharacters` counts characters, which for an astral run + * would be twice the units - so this is what the serialized sections use. + */ +function cutAtCodeUnits(value: string, maxUnits: number): string { + if (value.length <= maxUnits) return value; + + const isHighSurrogate = value.charCodeAt(maxUnits - 1) >= 0xd800 && value.charCodeAt(maxUnits - 1) <= 0xdbff; + return `${value.slice(0, isHighSurrogate ? maxUnits - 1 : maxUnits)}…`; +} + +/** + * Renders an untrusted value (header, rejected body field) for inclusion in a log line: it goes + * through {@link maskLogText} - masked, stripped of anything that could break the line, masked + * again - and is then capped. The masking runs before the cut, so a truncated email or wallet + * cannot slip through. + * + * Beyond `MAX_STRING` the value is reported by length instead: masking is regex work over the + * whole string, and the caller's cap alone would not stop an oversized one from paying for it. + * That length is in UTF-16 code units, the measure `MAX_STRING` is compared against and the one + * `String.length` gives for free - counting characters would mean walking the oversized string + * this branch exists to avoid, so the unit is named rather than converted. + */ +export function maskLogValue(value: string, maxLength: number): string { + if (value.length > MAX_STRING) return `<${value.length} code units>`; + + return capCharacters(maskLogText(value), maxLength); } // `budget` bounds the total work per section: each processed node deducts from @@ -62,7 +141,7 @@ function redact(value: unknown, key: string | undefined, budget: { left: number if (Buffer.isBuffer(value)) return ``; if (typeof value === 'string') { budget.left -= Math.min(value.length, MAX_STRING); - return value.length > MAX_STRING ? `<… ${value.length} chars …>` : maskValue(value); + return value.length > MAX_STRING ? `<… ${value.length} chars …>` : maskLogText(value); } if (value && typeof value === 'object') { const out: Record = {}; @@ -84,11 +163,14 @@ function format(value: unknown): string { try { // redact() handles Buffer + the array case (Array.isArray first), so the // raw value is never length/type-inspected here. - s = JSON.stringify(redact(value, undefined, { left: REDACT_BUDGET })); + // `JSON.stringify` escapes the control characters but leaves U+2028 / U+2029 as they are, so + // the serialized section is put through the same collapse as the free-form values above - it is + // what keeps the trace the single line the caller below documents. + s = singleLine(JSON.stringify(redact(value, undefined, { left: REDACT_BUDGET }))); } catch { return '(unserializable)'; } - return s.length > MAX_PART ? `${s.slice(0, MAX_PART)}…(${s.length} chars)` : s; + return s.length > MAX_PART ? `${cutAtCodeUnits(s, MAX_PART)}(${s.length} code units)` : s; } /** @@ -136,7 +218,10 @@ export function apiTraceMiddleware(): RequestHandler { res.on('finish', () => { const durationMs = Date.now() - start; const path = maskUrl(req.originalUrl); - const meta = `${req.method} ${path} → ${res.statusCode} (${durationMs}ms) client=${clientStr || '(none)'}`; + // The client header is the one free-form value on this line: it arrives from the caller, so + // it is rendered like every other one rather than interpolated as it came. + const client = maskLogValue(clientStr, MAX_CLIENT) || '(none)'; + const meta = `${req.method} ${path} → ${res.statusCode} (${durationMs}ms) client=${client}`; if (isRealUnitPath) { logger.info( `${meta} req.headers=${format(req.headers)} req.body=${format(req.body)} res.body=${format(responseBody)}`, diff --git a/src/shared/models/setting/__tests__/setting.service.spec.ts b/src/shared/models/setting/__tests__/setting.service.spec.ts index 43a5a44d63..e313a887a9 100644 --- a/src/shared/models/setting/__tests__/setting.service.spec.ts +++ b/src/shared/models/setting/__tests__/setting.service.spec.ts @@ -1,8 +1,9 @@ +import { ForbiddenException } from '@nestjs/common'; import { createMock } from '@golevelup/ts-jest'; import { Test, TestingModule } from '@nestjs/testing'; import { Setting } from '../setting.entity'; import { SettingRepository } from '../setting.repository'; -import { SettingService } from '../setting.service'; +import { SettingService, SystemManagedSettings } from '../setting.service'; describe('SettingService', () => { let service: SettingService; @@ -72,4 +73,62 @@ describe('SettingService', () => { await expect(service.getDeniedJwtAccounts()).resolves.toEqual([5, 6]); }); }); + + describe('set', () => { + it('writes an ordinary setting under the given key', async () => { + settingRepo.findOneBy.mockResolvedValue(null); + settingRepo.create.mockImplementation((data) => Object.assign(new Setting(), data)); + + await service.set('someOpsFlag', 'true'); + + // Pins key and value, not just that something was written: a swapped argument pair would + // otherwise pass, and this is the only place the pair is checked. + expect(settingRepo.save).toHaveBeenCalledWith(expect.objectContaining({ key: 'someOpsFlag', value: 'true' })); + }); + + // `staffKycClearance` decides who reaches every elevated endpoint and is derived from KYC data by a + // sync job. A manual write through the generic setter would grant elevated access to accounts that + // never passed KYC, and would stay live until the next sync run overwrote it. The check sits here + // rather than in the controller so every caller of `set` is covered, not just the HTTP route. + describe.each(SystemManagedSettings)('system-managed setting %s', (key) => { + it('is rejected, without writing', async () => { + await expect(service.set(key, '[1,2,3]')).rejects.toBeInstanceOf(ForbiddenException); + + expect(settingRepo.save).not.toHaveBeenCalled(); + }); + }); + + it('lists staffKycClearance as system-managed', () => { + expect(SystemManagedSettings).toContain('staffKycClearance'); + }); + + // The sync path writes through `setObj`, which goes to the repository directly — blocking it would + // freeze the allowlist at whatever it happened to contain. + it('still allows the sync path to write the clearance through setObj', async () => { + await service.setObj('staffKycClearance', [1, 2]); + + expect(settingRepo.save).toHaveBeenCalled(); + }); + }); + + describe('getStaffKycClearance', () => { + it('returns the cleared account ids', async () => { + mockSettings({ staffKycClearance: [1, 2] }); + + await expect(service.getStaffKycClearance()).resolves.toEqual([1, 2]); + }); + + it('coerces string ids to numbers', async () => { + mockSettings({ staffKycClearance: ['1', '2'] }); + + await expect(service.getStaffKycClearance()).resolves.toEqual([1, 2]); + }); + + // Fail-closed: a missing setting must read as "nobody is cleared", never as "no restriction". + it('returns an empty array when the setting is missing', async () => { + mockSettings({}); + + await expect(service.getStaffKycClearance()).resolves.toEqual([]); + }); + }); }); diff --git a/src/shared/models/setting/setting.service.ts b/src/shared/models/setting/setting.service.ts index 680ae01222..d4219c9a68 100644 --- a/src/shared/models/setting/setting.service.ts +++ b/src/shared/models/setting/setting.service.ts @@ -1,4 +1,4 @@ -import { BadRequestException, Injectable } from '@nestjs/common'; +import { BadRequestException, ForbiddenException, Injectable } from '@nestjs/common'; import { plainToInstance } from 'class-transformer'; import { validate } from 'class-validator'; import { Process } from 'src/shared/services/process.service'; @@ -8,6 +8,12 @@ import { isArraySchema, isPrimitiveSchema, SettingSchema, SettingSchemaRegistry import { Setting } from './setting.entity'; import { SettingRepository } from './setting.repository'; +// Settings whose value is derived by a sync job, never by an operator. The generic `PUT /setting/:key` +// route rejects them: for `staffKycClearance` a manual write would hand elevated access to accounts that +// never passed KYC — the very thing the gate exists to prevent — and would stay live until the next sync +// overwrites it. The sync path itself writes through `setObj` and is unaffected. +export const SystemManagedSettings = ['staffKycClearance']; + @Injectable() export class SettingService { constructor(private readonly settingRepo: SettingRepository) {} @@ -27,6 +33,11 @@ export class SettingService { } async set(key: string, value: string): Promise { + // Sync-owned settings are not writable through the generic setter — see SystemManagedSettings. + // `setObj` (the sync path) writes to the repository directly and is deliberately unaffected. + if (SystemManagedSettings.includes(key)) + throw new ForbiddenException(`Setting ${key} is maintained by the system and cannot be set manually`); + await this.validateSettingValue(key, value); const entity = (await this.settingRepo.findOneBy({ key })) ?? this.settingRepo.create({ key }); @@ -131,6 +142,15 @@ export class SettingService { return [...new Set([...manual, ...auto].map(Number))]; } + // Account (user data) ids cleared for elevated endpoints, maintained by StaffKycClearanceService. + // No manual-override counterpart on purpose: the clearance is a KYC fact, not an ops decision — an + // editable override would be a way to hand out admin access without the identification behind it. + // The generic `PUT /setting/:key` route would be exactly such an override, which is why the key is + // listed in `SystemManagedSettings` above and rejected by `set`. + async getStaffKycClearance(): Promise { + return this.getObj<(string | number)[]>('staffKycClearance', []).then((list) => list.map(Number)); + } + async getCustomBalanceSettings(): Promise<{ addresses: string[]; assets: string[] }> { const [addresses, assets] = await Promise.all([ this.getObjCached('customBalanceAddresses', []), diff --git a/src/shared/pipes/__tests__/detailed-validation.pipe.spec.ts b/src/shared/pipes/__tests__/detailed-validation.pipe.spec.ts new file mode 100644 index 0000000000..7053a8258d --- /dev/null +++ b/src/shared/pipes/__tests__/detailed-validation.pipe.spec.ts @@ -0,0 +1,277 @@ +import { + ArgumentMetadata, + BadRequestException, + HttpStatus, + UnprocessableEntityException, + ValidationPipe, +} from '@nestjs/common'; +import { Type } from 'class-transformer'; +import { + IsBoolean, + IsEnum, + IsIn, + IsInt, + IsNotEmptyObject, + IsOptional, + IsString, + IsUrl, + ValidateNested, + ValidationError, +} from 'class-validator'; +import { LogRejectedValue } from 'src/shared/decorators/log-rejected-value.decorator'; +import { + DetailedValidationPipe, + ValidationFailedException, + describeRejectedValues, +} from 'src/shared/pipes/detailed-validation.pipe'; + +enum TestMethod { + BANK = 'Bank', + CARD = 'Card', +} + +class NestedDto { + @IsString() + label: string; +} + +class TestDto { + @IsEnum(TestMethod) + @LogRejectedValue([...Object.values(TestMethod), 'Crypto']) + method: TestMethod; + + @IsOptional() + @IsInt() + amount: number; + + @IsOptional() + @IsString() + iban: string; + + @IsOptional() + @IsString() + wallet: string; + + @IsOptional() + @IsUrl() + webhookUrl: string; + + @IsOptional() + @IsBoolean() + @IsIn([true]) + @LogRejectedValue([true, false]) + confirmed: boolean; + + @IsOptional() + @IsNotEmptyObject() + @ValidateNested() + @Type(() => NestedDto) + nested: NestedDto; +} + +const metadata: ArgumentMetadata = { type: 'body', metatype: TestDto, data: '' }; +const options = { whitelist: true, transformOptions: { exposeUnsetFields: false } }; + +async function reject( + body: Record, + pipe: ValidationPipe = new DetailedValidationPipe(options), +): Promise { + try { + await pipe.transform(body, metadata); + } catch (error) { + return error; + } + + throw new Error('expected the body to be rejected'); +} + +async function rejectionDetail(body: Record): Promise { + const error = await reject(body); + expect(error).toBeInstanceOf(ValidationFailedException); + + return describeRejectedValues((error as ValidationFailedException).validationErrors); +} + +describe('DetailedValidationPipe', () => { + it('leaves a valid body untouched', async () => { + const dto = await new DetailedValidationPipe(options).transform({ method: 'Bank', amount: 5 }, metadata); + + expect(dto).toMatchObject({ method: TestMethod.BANK, amount: 5 }); + }); + + it('produces exactly the response body of the stock ValidationPipe', async () => { + const body = { method: 'Crypto', amount: 'ten', nested: { label: 42 } }; + + const detailed = (await reject(body)) as BadRequestException; + const stock = (await reject(body, new ValidationPipe(options))) as BadRequestException; + + expect(detailed.getStatus()).toBe(stock.getStatus()); + expect(detailed.getResponse()).toEqual(stock.getResponse()); + }); + + it('rejects with a BadRequestException that carries the raw validation errors', async () => { + const error = (await reject({ method: 'Crypto' })) as ValidationFailedException; + + expect(error).toBeInstanceOf(BadRequestException); + expect(error).toBeInstanceOf(ValidationFailedException); + expect(error.validationErrors.map((e) => e.property)).toEqual(['method']); + expect(error.validationErrors[0].value).toBe('Crypto'); + }); + + it('keeps the response of the stock pipe with disableErrorMessages, too', async () => { + const body = { method: 'Crypto' }; + const silentOptions = { ...options, disableErrorMessages: true }; + + const detailed = (await reject(body, new DetailedValidationPipe(silentOptions))) as BadRequestException; + const stock = (await reject(body, new ValidationPipe(silentOptions))) as BadRequestException; + + expect(detailed).toBeInstanceOf(BadRequestException); + expect(detailed.getResponse()).toEqual(stock.getResponse()); + expect(detailed.getResponse()).not.toHaveProperty('message', expect.any(Array)); + }); + + it('accepts the empty-error call the base signature allows', () => { + const exception = new DetailedValidationPipe(options).createExceptionFactory()(); + + expect(exception).toBeInstanceOf(ValidationFailedException); + expect((exception as ValidationFailedException).validationErrors).toEqual([]); + }); + + it('passes the exception through when the base factory builds a non-400', async () => { + const error = await reject( + { method: 'Crypto' }, + new DetailedValidationPipe({ ...options, errorHttpStatusCode: HttpStatus.UNPROCESSABLE_ENTITY }), + ); + + expect(error).toBeInstanceOf(UnprocessableEntityException); + expect(error).not.toBeInstanceOf(ValidationFailedException); + }); +}); + +describe('describeRejectedValues', () => { + it('names the value that was rejected — the constraint message only names the field', async () => { + await expect(rejectionDetail({ method: 'Crypto' })).resolves.toBe("method='Crypto'"); + }); + + it('distinguishes a missing value from an empty and a null one', async () => { + await expect(rejectionDetail({})).resolves.toBe('method=(missing)'); + await expect(rejectionDetail({ method: '' })).resolves.toBe("method=''"); + await expect(rejectionDetail({ method: null })).resolves.toBe('method=(null)'); + }); + + it('renders a declared non-string value', async () => { + await expect(rejectionDetail({ method: 'Bank', confirmed: false })).resolves.toBe("confirmed='false'"); + }); + + it('renders a nested failure with its path', async () => { + await expect(rejectionDetail({ method: 'Bank', nested: { label: 42 } })).resolves.toBe('nested.label='); + }); + + it('keeps the shape but not the content of a field that did not opt in', async () => { + // `amount` never declared its rejected values loggable, so its content stays out of the log + // however harmless it looks - and so does `wallet`, whose name says nothing either. + await expect(rejectionDetail({ method: 'Bank', amount: 'ten' })).resolves.toBe('amount='); + await expect(rejectionDetail({ method: 'Bank', wallet: 42 })).resolves.toBe('wallet='); + }); + + it('keeps a rejected URL out of the log, query string and all', async () => { + // A webhook or redirect target can carry a credential in its query string, and its field name + // says nothing about that. + const detail = await rejectionDetail({ method: 'Bank', webhookUrl: 'not-a-url?token=secret' }); + + expect(detail).not.toContain('secret'); + expect(detail).toBe('webhookUrl='); + }); + + it('redacts the value of a sensitive field by name', async () => { + const detail = await rejectionDetail({ method: 'Bank', iban: 42 }); + + expect(detail).toBe('iban=***'); + }); + + it('renders nothing for a value the field never declared, personal or not', async () => { + const detail = await rejectionDetail({ method: 'foo@bar.com' }); + + expect(detail).not.toContain('foo@bar.com'); + expect(detail).toBe('method='); + }); + + it('collapses control characters in the field name too', () => { + const error: ValidationError = { + property: 'evil\n2026-01-01 WARN forged', + value: 'x', + constraints: { isEnum: 'nope' }, + children: [], + target: new TestDto(), + }; + + expect(describeRejectedValues([error])).not.toContain('\n'); + }); + + it('renders nothing for a declared value a control character was appended to', async () => { + const detail = await rejectionDetail({ method: 'Crypto\n2026-01-01 WARN forged' }); + + expect(detail).not.toContain('\n'); + expect(detail).not.toContain('forged'); + expect(detail).toBe('method='); + }); + + it('summarizes a long value by length instead of rendering any of it', async () => { + await expect(rejectionDetail({ method: 'x'.repeat(100) })).resolves.toBe('method='); + await expect(rejectionDetail({ method: 'x'.repeat(600) })).resolves.toBe('method='); + }); + + it('keeps an account number a client put in a declaring field out of the log', async () => { + // What a validator accepts does not bound what a client sends: a field taking three payment + // methods rejects an account number as readily as a typo. Only a declared value is rendered, + // so what arrives outside the declaration never reaches the line. + const detail = await rejectionDetail({ method: 'CH9300762011623852957' }); + + expect(detail).toBe('method='); + }); + + it('renders the declared constant, not the string that arrived', async () => { + // Same constant, different case: what is written comes from the declaration either way. + await expect(rejectionDetail({ method: 'crypto' })).resolves.toBe("method='Crypto'"); + }); + + it('renders nothing for an error without a target', () => { + // A `ValidationError` built without the object it came from cannot answer for its fields. + const error: ValidationError = { property: 'method', value: 'Crypto', constraints: { isEnum: 'x' }, children: [] }; + + expect(describeRejectedValues([error])).toBe('method='); + }); + + it('renders nothing for a field whose declaration does not hold the value', async () => { + await expect(rejectionDetail({ method: 'Bank', confirmed: 'yes' })).resolves.toBe('confirmed='); + }); + + it('summarizes structured values instead of dumping the body', async () => { + await expect(rejectionDetail({ method: ['Bank'] })).resolves.toBe('method='); + await expect(rejectionDetail({ method: { a: 1 } })).resolves.toBe('method='); + }); + + it('stops at the depth cap and marks the rendering as incomplete', () => { + // Five levels, failing at the deepest one: the walk stops at the cap and never reaches it. + const deepest: ValidationError = { property: 'e', value: 1, constraints: { isEnum: 'nope' }, children: [] }; + const nested = ['d', 'c', 'b', 'a'].reduce( + (child, property) => ({ property, children: [child] }), + deepest, + ); + + expect(describeRejectedValues([nested])).toBe('…'); + }); + + it('bounds the number of rendered fields and marks the rendering as incomplete', () => { + const errors: ValidationError[] = Array.from({ length: 8 }, (_, i) => ({ + property: `field${i}`, + value: i, + constraints: { isEnum: 'nope' }, + children: [], + })); + + const detail = describeRejectedValues(errors); + + expect(detail).toBe('field0=, field1=, field2=, field3=, field4=, …'); + }); +}); diff --git a/src/shared/pipes/detailed-validation.pipe.ts b/src/shared/pipes/detailed-validation.pipe.ts new file mode 100644 index 0000000000..a47259ff43 --- /dev/null +++ b/src/shared/pipes/detailed-validation.pipe.ts @@ -0,0 +1,135 @@ +import { BadRequestException, ValidationError, ValidationPipe } from '@nestjs/common'; +import { loggableRejectedValues } from 'src/shared/decorators/log-rejected-value.decorator'; +import { REDACT_KEY, maskLogValue } from 'src/shared/middlewares/api-trace.middleware'; + +// Fields listed per rejection, and the cap per rendered value. Both are small on purpose: this is +// a diagnostic hint for the log line, not a body dump. +const MAX_FIELDS = 5; +const MAX_VALUE_LENGTH = 64; +const MAX_DEPTH = 3; + +/** + * A failed request-body validation, carrying the raw `ValidationError[]` alongside the response. + * + * The response body is the one the stock `ValidationPipe` would have produced — this only keeps the + * errors reachable for the log line in `ApiExceptionFilter`, which is where the rejected *value* + * becomes visible. The constraint messages in the body name the field and the accepted values, not + * the value that arrived, so a wrong constant in a client cannot be named from the logs otherwise. + */ +export class ValidationFailedException extends BadRequestException { + constructor( + response: Record, + readonly validationErrors: ValidationError[], + ) { + super(response); + } +} + +/** + * `ValidationPipe` that raises a {@link ValidationFailedException} instead of a plain + * `BadRequestException`. The response is unchanged: the exception is built by the base factory and + * only re-wrapped, so status, message array and body shape are byte-identical to the stock pipe. + */ +export class DetailedValidationPipe extends ValidationPipe { + createExceptionFactory(): (errors?: ValidationError[]) => unknown { + const createException = super.createExceptionFactory(); + + return (errors: ValidationError[] = []) => { + const exception = createException(errors); + + // Anything but the 400 is passed through: with `errorHttpStatusCode` set, the base factory + // builds a different exception class. + if (!(exception instanceof BadRequestException)) return exception; + + // The base factory composes an object body, and `HttpException.createBody` passes an object + // through verbatim — that is what keeps the re-raised exception identical. Pinned by the + // specs that compare the response against a stock `ValidationPipe` for the same body, which + // is where a change to that would surface. + return new ValidationFailedException(exception.getResponse() as Record, errors); + }; + } +} + +/** + * Renders what was rejected as `field=value` pairs for a log line. What arrived is untrusted input + * and never reaches the line: a value is rendered only where the field declared the set it may come + * from, and what is written is that declared constant. Every other field is reduced to its shape, + * the field name is masked and rendered single-line like any other value from the request, and the + * list itself is bounded in count and depth. + */ +export function describeRejectedValues(errors: ValidationError[]): string { + const fields: string[] = []; + const complete = collectRejectedValues(errors, '', 0, fields); + + return (complete ? fields : [...fields, '…']).join(', '); +} + +// Returns false if the walk was cut short (field cap or depth cap), so the caller can mark the +// rendering as incomplete rather than implying the list is everything that was rejected. +function collectRejectedValues(errors: ValidationError[], prefix: string, depth: number, fields: string[]): boolean { + for (const error of errors) { + if (fields.length >= MAX_FIELDS) return false; + + // The field name also goes through `maskLogValue`, like a rendered string value does: it is a + // property of the parsed body, so a DTO that validates through a client-keyed object would put + // the client in charge of it, and this covers that too. + const property = maskLogValue(`${error.property}`, MAX_VALUE_LENGTH); + const path = prefix ? `${prefix}.${property}` : property; + if (error.constraints) fields.push(`${path}=${renderValue(error)}`); + + if (error.children?.length) { + if (depth + 1 > MAX_DEPTH) return false; + if (!collectRejectedValues(error.children, path, depth + 1, fields)) return false; + } + } + + return true; +} + +function renderValue(error: ValidationError): string { + const { property, value } = error; + + // Absent is the one thing worth naming for every field: there is nothing to disclose, and it is + // what separates "the client never sent this" from "the client sent the wrong thing". + if (value === undefined) return '(missing)'; + if (value === null) return '(null)'; + if (value === '') return "''"; + + // A declared match comes ahead of the name-based redaction: what it renders is a constant of this + // program, so the field's name says nothing about it. `personalIbanProvider` is the case - the + // name carries `iban` and would otherwise lose a value that was never the client's to begin with. + const declared = renderDeclared(error); + if (declared !== undefined) return declared; + + if (REDACT_KEY.test(property)) return '***'; + + return summarize(value); +} + +// A value is rendered only if the field declared it (see {@link LogRejectedValue}) - and what is +// written is the declared constant, not the string that arrived, so nothing the request composed +// reaches the line even where the two differ in case. Every other value keeps its shape and loses +// its content, which is what bounds this: what a validator accepts says nothing about what a client +// sends, and a rejected value is by definition outside what was accepted. +// +// The declaration is read from the object being validated rather than passed down, so a nested DTO +// answers for its own fields. A `ValidationError` built without a target declares nothing. +function renderDeclared(error: ValidationError): string | undefined { + if (typeof error.value !== 'string' && typeof error.value !== 'number' && typeof error.value !== 'boolean') { + return undefined; + } + + const declared = loggableRejectedValues(error.target?.constructor, error.property); + const match = declared?.get(`${error.value}`.toLowerCase()); + + // The constant comes from this code rather than from the request, but it is rendered like every + // other value on the line - a declaration is written by hand, and nothing here has to trust that. + return match === undefined ? undefined : `'${maskLogValue(match, MAX_VALUE_LENGTH)}'`; +} + +function summarize(value: unknown): string { + if (typeof value === 'string') return ``; + if (Array.isArray(value)) return ``; + + return `<${typeof value}>`; +} diff --git a/src/shared/services/__tests__/process.service.spec.ts b/src/shared/services/__tests__/process.service.spec.ts index daf3feb509..45230e301a 100644 --- a/src/shared/services/__tests__/process.service.spec.ts +++ b/src/shared/services/__tests__/process.service.spec.ts @@ -1,3 +1,4 @@ +import { HasStaffKycClearance, SetStaffKycClearance } from 'src/shared/auth/staff-kyc-clearance'; import { SettingService } from 'src/shared/models/setting/setting.service'; import { IsJwtAccountDenied, ProcessService } from 'src/shared/services/process.service'; @@ -47,3 +48,38 @@ describe('ProcessService JWT account denylist', () => { expect(IsJwtAccountDenied(123)).toBe(false); }); }); + +describe('ProcessService staff KYC clearance priming', () => { + let settingService: jest.Mocked; + let service: ProcessService; + + beforeEach(() => { + settingService = { + getStaffKycClearance: jest.fn().mockResolvedValue([]), + } as unknown as jest.Mocked; + + service = new ProcessService(settingService); + }); + + afterEach(() => SetStaffKycClearance([])); + + it('primes the in-memory clearance Set from the setting', async () => { + settingService.getStaffKycClearance.mockResolvedValue([123, 456]); + + await service.resyncStaffKycClearance(); + + expect(HasStaffKycClearance(123)).toBe(true); + expect(HasStaffKycClearance(456)).toBe(true); + expect(HasStaffKycClearance(999)).toBe(false); + }); + + it('drops a revoked account on the next resync', async () => { + settingService.getStaffKycClearance.mockResolvedValue([123]); + await service.resyncStaffKycClearance(); + + settingService.getStaffKycClearance.mockResolvedValue([]); + await service.resyncStaffKycClearance(); + + expect(HasStaffKycClearance(123)).toBe(false); + }); +}); diff --git a/src/shared/services/process.service.ts b/src/shared/services/process.service.ts index 97d79a09b5..4a9f1ba2e5 100644 --- a/src/shared/services/process.service.ts +++ b/src/shared/services/process.service.ts @@ -1,6 +1,7 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; import { Config } from 'src/config/config'; +import { SetStaffKycClearance } from 'src/shared/auth/staff-kyc-clearance'; import { SettingService } from '../models/setting/setting.service'; import { DfxCron } from '../utils/cron'; @@ -174,6 +175,9 @@ export class ProcessService implements OnModuleInit { // is fail-closed by sentinel await this.resyncDeniedJwtAddresses(); await this.resyncDeniedJwtAccounts(); + // await as well, but for the opposite reason: the staff clearance Set is fail-closed, so serving + // HTTP before it is primed would deny every elevated endpoint instead of over-granting. + await this.resyncStaffKycClearance(); } @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800 }) @@ -198,6 +202,13 @@ export class ProcessService implements OnModuleInit { DeniedJwtAccounts = new Set(list); } + // Primes the fail-closed staff clearance allowlist — see `staff-kyc-clearance.ts` for the semantics. + @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800 }) + async resyncStaffKycClearance(): Promise { + const list = await this.settingService.getStaffKycClearance(); + SetStaffKycClearance(list); + } + public async setSafetyModeActive(active: boolean): Promise { this.safetyModeInactive = DisabledProcess(Process.SAFETY_MODE) ? true : !active; await this.resyncDisabledProcesses(); diff --git a/src/shared/utils/__tests__/request-caller.spec.ts b/src/shared/utils/__tests__/request-caller.spec.ts new file mode 100644 index 0000000000..f4b9069d92 --- /dev/null +++ b/src/shared/utils/__tests__/request-caller.spec.ts @@ -0,0 +1,92 @@ +import { Request } from 'express'; +import { describeCaller } from 'src/shared/utils/request-caller'; + +describe('describeCaller', () => { + const req = (headers: Record): Request => ({ headers }) as unknown as Request; + + it('reports the X-Client value', () => { + expect(describeCaller(req({ 'x-client': 'dfx-services' }))).toBe('client=dfx-services'); + }); + + it('reports an absent client explicitly, so an unattributable caller is visible as such', () => { + expect(describeCaller(req({}))).toBe('client=(none)'); + }); + + it('adds the requesting site and the user agent', () => { + const caller = describeCaller( + req({ 'x-client': 'dfx-services', origin: 'https://app.dfx.swiss', 'user-agent': 'Mozilla/5.0 (X11)' }), + ); + + expect(caller).toBe('client=dfx-services origin=https://app.dfx.swiss ua=Mozilla/5.0 (X11)'); + }); + + it('reduces the origin header to its origin too — it arrives from the client like the rest', () => { + const caller = describeCaller(req({ origin: 'https://partner.example.com/checkout?token=secret' })); + + expect(caller).toBe('client=(none) origin=https://partner.example.com'); + }); + + it('falls back to the origin of the referer, dropping its path and query', () => { + const caller = describeCaller(req({ referer: 'https://partner.example.com/checkout?token=secret&mail=a@b.ch' })); + + expect(caller).toContain('origin=https://partner.example.com'); + expect(caller).not.toContain('secret'); + expect(caller).not.toContain('checkout'); + }); + + it('prefers Origin over Referer', () => { + const caller = describeCaller(req({ origin: 'https://a.example.com', referer: 'https://b.example.com/x' })); + + expect(caller).toContain('origin=https://a.example.com'); + expect(caller).not.toContain('b.example.com'); + }); + + it('drops an unparsable value rather than logging it raw', () => { + expect(describeCaller(req({ referer: 'not a url' }))).toBe('client=(none)'); + expect(describeCaller(req({ origin: 'not a url' }))).toBe('client=(none)'); + }); + + it('keeps an opaque origin — having none to name says something too', () => { + expect(describeCaller(req({ origin: 'null' }))).toBe('client=(none) origin=null'); + // It yields an origin, so it wins over the referer like any other origin would. + expect(describeCaller(req({ origin: 'null', referer: 'https://partner.example.com/x' }))).toBe( + 'client=(none) origin=null', + ); + }); + + it('falls back to the referer when the origin yields nothing, not just when it is absent', () => { + const caller = describeCaller(req({ origin: 'not a url', referer: 'https://partner.example.com/x?t=1' })); + + expect(caller).toBe('client=(none) origin=https://partner.example.com'); + }); + + it('takes the first value of a tampered array header', () => { + expect(describeCaller(req({ 'x-client': ['dfx-services', 'other'] }))).toBe('client=dfx-services'); + expect(describeCaller(req({ 'x-client': [], origin: [] }))).toBe('client=(none)'); + }); + + it('collapses control characters, so a header cannot forge a second log line', () => { + const caller = describeCaller(req({ 'x-client': 'a\n2026-01-01 WARN forged' })); + + expect(caller).not.toContain('\n'); + }); + + it('caps each header, so an oversized one cannot flood the log line', () => { + // The origin has to be a parsable URL to reach the cap at all - an unparsable one is dropped + // by `callerOrigin` before it gets there, which would leave the cap untested. + const caller = describeCaller( + req({ + 'x-client': 'c'.repeat(500), + origin: `https://${'a'.repeat(100)}.example.com`, + 'user-agent': 'u'.repeat(500), + }), + ); + + expect(caller).toContain(`origin=https://${'a'.repeat(56)}\u2026`); + expect(caller.length).toBeLessThan(250); + }); + + it('survives a request without headers — the filter must not turn a 400 into a 500', () => { + expect(describeCaller({} as Request)).toBe('client=(none)'); + }); +}); diff --git a/src/shared/utils/request-caller.ts b/src/shared/utils/request-caller.ts new file mode 100644 index 0000000000..cba7cdfa45 --- /dev/null +++ b/src/shared/utils/request-caller.ts @@ -0,0 +1,70 @@ +import { Request } from 'express'; +import { maskLogValue } from 'src/shared/middlewares/api-trace.middleware'; +import { getClient } from 'src/shared/utils/request-client'; + +// Caps per header. All three are client-supplied and unauthenticated (see the note in +// `request-client.ts`): they are a diagnostic hint about who is calling, never an identity. +const MAX_CLIENT_LENGTH = 32; +const MAX_ORIGIN_LENGTH = 64; +const MAX_USER_AGENT_LENGTH = 96; + +/** + * Renders what a request says about its caller — `X-Client`, the requesting site, and the user + * agent — for a log line. + * + * On an endpoint that runs without authentication these headers are all a log line has to go on: + * without them, a partner integration, one of our own apps and a third-party script are the same + * anonymous caller. Of the requesting URL only the origin is used — never its path or query, which + * can carry personal data or tokens — so a browser-side caller can be named by site. + */ +export function describeCaller(req: Request): string { + // The exception filter is the last line of defence: a request object without headers (a + // non-HTTP execution context) must not turn a rejected request into a 500 in here. + if (!req?.headers) return 'client=(none)'; + + const parts = [`client=${maskLogValue(getClient(req), MAX_CLIENT_LENGTH) || '(none)'}`]; + + const origin = callerOrigin(req); + if (origin) parts.push(`origin=${maskLogValue(origin, MAX_ORIGIN_LENGTH)}`); + + const userAgent = firstHeader(req, 'user-agent'); + if (userAgent) parts.push(`ua=${maskLogValue(userAgent, MAX_USER_AGENT_LENGTH)}`); + + return parts.join(' '); +} + +// Both headers are reduced to their origin, `Origin` included: it is supposed to carry nothing +// else, but it arrives from the client like everything here, and a value that is not what it is +// supposed to be is exactly the one that must not reach the log with a query string attached. +// +// The first header that yields an origin wins, not the first one that is present: an `Origin` the +// client filled with something else should cost its own attribution, not the `Referer`'s too. +function callerOrigin(req: Request): string { + for (const header of ['origin', 'referer']) { + const origin = toOrigin(firstHeader(req, header)); + if (origin) return origin; + } + + return ''; +} + +function toOrigin(url: string): string { + // What a browser sends for an opaque origin — a sandboxed frame, a redirect across sites. It is + // not a URL and cannot be reduced to one, but "the caller has no origin to name" is itself worth + // the line, and it is a fixed word rather than anything the client composed. + if (url === 'null') return url; + + try { + return new URL(url).origin; + } catch { + // Not a parsable URL — dropped rather than logged raw, since the unparsed value would be the + // one that is not reduced to its origin. + return ''; + } +} + +// A tampered header can arrive as an array (CodeQL js/type-confusion-through-parameter-tampering). +function firstHeader(req: Request, name: string): string { + const value = req.headers[name]; + return ((Array.isArray(value) ? value[0] : value) ?? '').trim(); +} diff --git a/src/subdomains/core/buy-crypto/routes/buy/dto/__tests__/get-buy-payment-info.dto.spec.ts b/src/subdomains/core/buy-crypto/routes/buy/dto/__tests__/get-buy-payment-info.dto.spec.ts index 7af3cf6b34..eb337f05c3 100644 --- a/src/subdomains/core/buy-crypto/routes/buy/dto/__tests__/get-buy-payment-info.dto.spec.ts +++ b/src/subdomains/core/buy-crypto/routes/buy/dto/__tests__/get-buy-payment-info.dto.spec.ts @@ -1,12 +1,17 @@ -import { ArgumentMetadata, BadRequestException, ValidationPipe } from '@nestjs/common'; +import { ArgumentMetadata, BadRequestException } from '@nestjs/common'; +import { + DetailedValidationPipe, + ValidationFailedException, + describeRejectedValues, +} from 'src/shared/pipes/detailed-validation.pipe'; import { FiatPaymentMethod } from 'src/subdomains/supporting/payment/dto/payment-method.enum'; import { QuoteError } from 'src/subdomains/supporting/payment/dto/transaction-helper/quote-error.enum'; import { GetBuyPaymentInfoDto, PersonalIbanProvider } from '../get-buy-payment-info.dto'; describe('GetBuyPaymentInfoDto.personalIbanProvider', () => { // Mirror the production global pipe (src/main.ts): custom decorator messages must surface as-is - // in the 400 body (no exceptionFactory override). - const pipe = new ValidationPipe({ whitelist: true, transformOptions: { exposeUnsetFields: false } }); + // in the 400 body. + const pipe = new DetailedValidationPipe({ whitelist: true, transformOptions: { exposeUnsetFields: false } }); const metadata: ArgumentMetadata = { type: 'body', metatype: GetBuyPaymentInfoDto, data: '' }; const validBody = { @@ -40,4 +45,33 @@ describe('GetBuyPaymentInfoDto.personalIbanProvider', () => { expect(messageText).toContain(QuoteError.PERSONAL_IBAN_PROVIDER_UNSUPPORTED); expect(messageText).not.toMatch(/must be one of the following values/i); }); + + it('names a provider that differs only in case, despite the field name carrying `iban`', async () => { + // The declared value wins over the name-based redaction: what it renders is this program's + // constant, and the message alone would not say which provider the client meant. + let caught: unknown; + try { + await pipe.transform({ ...validBody, personalIbanProvider: 'frick' }, metadata); + } catch (error) { + caught = error; + } + + expect(caught).toBeInstanceOf(ValidationFailedException); + expect(describeRejectedValues((caught as ValidationFailedException).validationErrors)).toBe( + "personalIbanProvider='Frick'", + ); + }); + + it('still redacts a provider value the field never declared', async () => { + let caught: unknown; + try { + await pipe.transform({ ...validBody, personalIbanProvider: 'CH9300762011623852957' }, metadata); + } catch (error) { + caught = error; + } + + expect(describeRejectedValues((caught as ValidationFailedException).validationErrors)).toBe( + 'personalIbanProvider=***', + ); + }); }); diff --git a/src/subdomains/core/buy-crypto/routes/buy/dto/__tests__/get-buy-quote.dto.spec.ts b/src/subdomains/core/buy-crypto/routes/buy/dto/__tests__/get-buy-quote.dto.spec.ts new file mode 100644 index 0000000000..45c1278e74 --- /dev/null +++ b/src/subdomains/core/buy-crypto/routes/buy/dto/__tests__/get-buy-quote.dto.spec.ts @@ -0,0 +1,51 @@ +import { ArgumentMetadata } from '@nestjs/common'; +import { + DetailedValidationPipe, + ValidationFailedException, + describeRejectedValues, +} from 'src/shared/pipes/detailed-validation.pipe'; +import { GetBuyQuoteDto } from '../get-buy-quote.dto'; + +// The 400 body names the field and, where the field declares them, the accepted values - never the +// value that arrived. These cases pin what the log line shows instead, for the two rejections +// covered below. +describe('GetBuyQuoteDto rejections', () => { + // Mirrors the production global pipe (src/main.ts). + const pipe = new DetailedValidationPipe({ whitelist: true, transformOptions: { exposeUnsetFields: false } }); + const metadata: ArgumentMetadata = { type: 'body', metatype: GetBuyQuoteDto, data: '' }; + + const body = { currency: { id: 1 }, asset: { id: 1 } }; + + async function rejectionDetail(input: Record): Promise { + try { + await pipe.transform(input, metadata); + } catch (error) { + expect(error).toBeInstanceOf(ValidationFailedException); + return describeRejectedValues((error as ValidationFailedException).validationErrors); + } + + throw new Error('expected the body to be rejected'); + } + + it('names the payment method that was sent, not just the accepted ones', async () => { + // 'Crypto' is a value of the wider `PaymentMethod` union (payment-method.enum.ts); this DTO + // accepts `FiatPaymentMethod` only. + await expect(rejectionDetail({ ...body, amount: 100, paymentMethod: 'Crypto' })).resolves.toBe( + "paymentMethod='Crypto'", + ); + }); + + it('distinguishes both amounts missing from both amounts set', async () => { + await expect(rejectionDetail({ ...body })).resolves.toBe('amount=(missing), targetAmount=(missing)'); + await expect(rejectionDetail({ ...body, amount: 100, targetAmount: 1 })).resolves.toBe( + 'amount=, targetAmount=', + ); + }); + + it('accepts a valid body unchanged', async () => { + const dto = await pipe.transform({ ...body, amount: 100, paymentMethod: 'Bank' }, metadata); + + expect(dto.amount).toBe(100); + expect(dto.paymentMethod).toBe('Bank'); + }); +}); diff --git a/src/subdomains/core/buy-crypto/routes/buy/dto/get-buy-payment-info.dto.ts b/src/subdomains/core/buy-crypto/routes/buy/dto/get-buy-payment-info.dto.ts index 7a388d9ca1..8813cd99bf 100644 --- a/src/subdomains/core/buy-crypto/routes/buy/dto/get-buy-payment-info.dto.ts +++ b/src/subdomains/core/buy-crypto/routes/buy/dto/get-buy-payment-info.dto.ts @@ -13,13 +13,14 @@ import { ValidateNested, } from 'class-validator'; import { EntityDto } from 'src/shared/dto/entity.dto'; +import { LogRejectedValue } from 'src/shared/decorators/log-rejected-value.decorator'; import { Asset } from 'src/shared/models/asset/asset.entity'; import { AssetInDto } from 'src/shared/models/asset/dto/asset.dto'; import { Fiat } from 'src/shared/models/fiat/fiat.entity'; import { Util } from 'src/shared/utils/util'; import { XOR } from 'src/shared/validators/xor.validator'; import { IbanType, IsDfxIban } from 'src/subdomains/supporting/bank/bank-account/is-dfx-iban.validator'; -import { FiatPaymentMethod } from 'src/subdomains/supporting/payment/dto/payment-method.enum'; +import { FiatPaymentMethod, PaymentMethodSwagger } from 'src/subdomains/supporting/payment/dto/payment-method.enum'; import { QuoteError } from 'src/subdomains/supporting/payment/dto/transaction-helper/quote-error.enum'; import { PersonalIbanProvider } from './personal-iban-provider.enum'; @@ -61,6 +62,8 @@ export class GetBuyPaymentInfoDto { @IsNotEmpty() @IsEnum(FiatPaymentMethod) + // The crypto members of the union are what a client sends here by mistake. + @LogRejectedValue(PaymentMethodSwagger) paymentMethod: FiatPaymentMethod = FiatPaymentMethod.BANK; @ApiPropertyOptional({ @@ -69,6 +72,8 @@ export class GetBuyPaymentInfoDto { }) @IsOptional() @IsEnum(PersonalIbanProvider, { message: QuoteError.PERSONAL_IBAN_PROVIDER_UNSUPPORTED }) + // The field's own values: a wrong one that differs only in case is named back as the constant. + @LogRejectedValue(PersonalIbanProvider) personalIbanProvider?: PersonalIbanProvider; @ApiPropertyOptional({ description: 'Custom transaction id' }) diff --git a/src/subdomains/core/buy-crypto/routes/buy/dto/get-buy-quote.dto.ts b/src/subdomains/core/buy-crypto/routes/buy/dto/get-buy-quote.dto.ts index 38017e0500..572137dbdb 100644 --- a/src/subdomains/core/buy-crypto/routes/buy/dto/get-buy-quote.dto.ts +++ b/src/subdomains/core/buy-crypto/routes/buy/dto/get-buy-quote.dto.ts @@ -11,12 +11,13 @@ import { ValidateIf, ValidateNested, } from 'class-validator'; +import { LogRejectedValue } from 'src/shared/decorators/log-rejected-value.decorator'; import { Asset } from 'src/shared/models/asset/asset.entity'; import { AssetInDto } from 'src/shared/models/asset/dto/asset.dto'; import { Fiat } from 'src/shared/models/fiat/fiat.entity'; import { FiatInDto } from 'src/shared/models/fiat/dto/fiat.dto'; import { XOR } from 'src/shared/validators/xor.validator'; -import { FiatPaymentMethod } from 'src/subdomains/supporting/payment/dto/payment-method.enum'; +import { FiatPaymentMethod, PaymentMethodSwagger } from 'src/subdomains/supporting/payment/dto/payment-method.enum'; export class GetBuyQuoteDto { @ApiProperty({ type: FiatInDto, description: 'Source currency (by ID or name)' }) @@ -48,6 +49,8 @@ export class GetBuyQuoteDto { @ApiPropertyOptional({ description: 'Payment method', enum: FiatPaymentMethod }) @IsNotEmpty() @IsEnum(FiatPaymentMethod) + // The crypto members of the union are what a client sends here by mistake. + @LogRejectedValue(PaymentMethodSwagger) paymentMethod: FiatPaymentMethod = FiatPaymentMethod.BANK; @ApiPropertyOptional({ description: 'This field is deprecated, use "specialCode" instead.', deprecated: true }) diff --git a/src/subdomains/core/history/services/__tests__/history-access.service.spec.ts b/src/subdomains/core/history/services/__tests__/history-access.service.spec.ts index a10c60a79f..71627fe124 100644 --- a/src/subdomains/core/history/services/__tests__/history-access.service.spec.ts +++ b/src/subdomains/core/history/services/__tests__/history-access.service.spec.ts @@ -1,6 +1,7 @@ import { createMock } from '@golevelup/ts-jest'; import { ForbiddenException, UnauthorizedException } from '@nestjs/common'; import { JwtPayload } from 'src/shared/auth/jwt-payload.interface'; +import { SetStaffKycClearance } from 'src/shared/auth/staff-kyc-clearance'; import { UserRole } from 'src/shared/auth/user-role.enum'; import { UserData } from 'src/subdomains/generic/user/models/user-data/user-data.entity'; import { UserDataService } from 'src/subdomains/generic/user/models/user-data/user-data.service'; @@ -164,6 +165,11 @@ describe('HistoryAccessService', () => { const ownerJwt: JwtPayload = { role: UserRole.USER, ip: '1.1.1.1', account: 1, user: 10, address: '0xAAA' }; const otherJwt: JwtPayload = { role: UserRole.USER, ip: '1.1.1.1', account: 2, user: 20, address: '0xCCC' }; + // Staff full access now additionally requires KYC clearance for the calling account; account 99 is + // the cleared staff account used by the staff cases below. + beforeEach(() => SetStaffKycClearance([99])); + afterEach(() => SetStaffKycClearance([])); + it('denies full view without JWT', () => { const tx = { userData: { id: 1 } } as Transaction; expect(service.canViewFullTransaction(undefined, tx)).toBe(false); @@ -206,6 +212,27 @@ describe('HistoryAccessService', () => { expect(service.canViewFullTransaction(staff, tx)).toBe(true); }); + it.each([UserRole.SUPPORT, UserRole.COMPLIANCE, UserRole.ADMIN])( + 'denies %s staff full access without KYC clearance', + (role) => { + SetStaffKycClearance([]); + const staff: JwtPayload = { role, ip: '1.1.1.1', account: 99 }; + const tx = { userData: { id: 1 } } as Transaction; + + // These routes are OptionalJwtAuthGuard-only, so this predicate is the only place the staff KYC + // gate can apply — without it, an uncleared admin keeps blanket access to every customer's tx. + expect(service.canViewFullTransaction(staff, tx)).toBe(false); + }, + ); + + it('still allows an uncleared staff member to view their OWN transaction', () => { + SetStaffKycClearance([]); + const staff: JwtPayload = { role: UserRole.ADMIN, ip: '1.1.1.1', account: 99 }; + const tx = { userData: { id: 99 } } as Transaction; + + expect(service.canViewFullTransaction(staff, tx)).toBe(true); + }); + it('denies REALUNIT ownership-independent full access (isolated external tenant, not DFX staff)', () => { const tenantStaff: JwtPayload = { role: UserRole.REALUNIT, ip: '1.1.1.1', account: 99 }; const tx = { userData: { id: 1 } } as Transaction; // not owned by the RealUnit tenant account diff --git a/src/subdomains/core/history/services/history-access.service.ts b/src/subdomains/core/history/services/history-access.service.ts index a55f976950..b8ec9325f1 100644 --- a/src/subdomains/core/history/services/history-access.service.ts +++ b/src/subdomains/core/history/services/history-access.service.ts @@ -1,6 +1,6 @@ import { ForbiddenException, Injectable, UnauthorizedException } from '@nestjs/common'; import { JwtPayload } from 'src/shared/auth/jwt-payload.interface'; -import { hasRoleAccess } from 'src/shared/auth/role.guard'; +import { hasStaffAccess } from 'src/shared/auth/role.guard'; import { UserRole } from 'src/shared/auth/user-role.enum'; import { UserData } from 'src/subdomains/generic/user/models/user-data/user-data.entity'; import { UserDataService } from 'src/subdomains/generic/user/models/user-data/user-data.service'; @@ -55,7 +55,7 @@ export class HistoryAccessService { */ canViewFullTransaction(jwt: JwtPayload | undefined, tx: Transaction | TransactionRequest | undefined): boolean { if (!jwt?.role) return false; - if (this.isStaffFullAccess(jwt.role)) return true; + if (this.isStaffFullAccess(jwt)) return true; return this.isOwner(jwt, tx); } @@ -65,12 +65,16 @@ export class HistoryAccessService { return accountId != null && accountId === jwt.account; } - private isStaffFullAccess(role: UserRole): boolean { - // DFX staff only: the SUPPORT hierarchy (COMPLIANCE / ADMIN / SUPER_ADMIN via hasRoleAccess). REALUNIT is an + private isStaffFullAccess(jwt: JwtPayload): boolean { + // DFX staff only: the SUPPORT hierarchy (COMPLIANCE / ADMIN / SUPER_ADMIN via hasStaffAccess). REALUNIT is an // isolated external tenant, NOT DFX staff — granting it ownership-independent full access here would leak every // customer's private banking/compliance data across the tenant boundary that its own routes scope via // RealUnitScopeService. RealUnit access to a customer's transaction must stay customer-scoped, never blanket. - return hasRoleAccess(UserRole.SUPPORT, role); + // + // `hasStaffAccess`, not `hasRoleAccess`: these routes are OptionalJwtAuthGuard-only, so no RoleGuard has + // applied the staff KYC gate. Ownership-independent access to every customer's history is exactly the + // privilege the gate exists for. + return hasStaffAccess(UserRole.SUPPORT, jwt); } private accountIdOf(tx: Transaction | TransactionRequest): number | undefined { diff --git a/src/subdomains/core/liquidity-management/adapters/actions/__tests__/scrypt.adapter.spec.ts b/src/subdomains/core/liquidity-management/adapters/actions/__tests__/scrypt.adapter.spec.ts index 03f2765ea9..482ea0f089 100644 --- a/src/subdomains/core/liquidity-management/adapters/actions/__tests__/scrypt.adapter.spec.ts +++ b/src/subdomains/core/liquidity-management/adapters/actions/__tests__/scrypt.adapter.spec.ts @@ -2,6 +2,7 @@ import { createMock } from '@golevelup/ts-jest'; import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.enum'; import { ScryptBalanceTransaction, + ScryptCancellation, ScryptOrderInfo, ScryptOrderStatus, ScryptTransactionStatus, @@ -59,8 +60,8 @@ function createUncertainSellOrder(overrides: Partial = } /** Minimal but fully typed venue order record, so the tests do not have to widen the return type. */ -function venueOrder(id: string, status = ScryptOrderStatus.NEW): ScryptOrderInfo { - return { id, symbol: 'EUR/USDT', side: 'Sell', status, quantity: 1, filledQuantity: 0, remainingQuantity: 1 }; +function venueOrder(id: string, status = ScryptOrderStatus.NEW, symbol = 'EUR/USDT'): ScryptOrderInfo { + return { id, symbol, side: 'Sell', status, quantity: 1, filledQuantity: 0, remainingQuantity: 1 }; } /** Typed action stub — `paramMap` is a getter over `params`, so the raw field is what a fixture sets. */ @@ -68,6 +69,15 @@ function withdrawAction(): LiquidityManagementAction { return Object.assign(new LiquidityManagementAction(), { command: ScryptAdapterCommands.WITHDRAW, params: '{}' }); } +/** Same as {@link withdrawAction}, for the deliberately unregistered command the reconciliation path must + * still handle by name alone (it resolves by system, not by a known command list). */ +function sellIfDeficitAction(paramMap: Record = {}): LiquidityManagementAction { + return Object.assign(new LiquidityManagementAction(), { + command: 'sell-if-deficit', + params: JSON.stringify(paramMap), + }); +} + describe('ScryptAdapter', () => { let adapter: ScryptAdapter; let scryptService: ScryptService; @@ -181,7 +191,8 @@ describe('ScryptAdapter', () => { it('keeps waiting on an aged withdrawal the venue DOES know but has not settled', async () => { // a record without a hash is an observation, not an unknown outcome — quarantining it would only - // bounce the order between reconciliation and the completion check + // bounce the order between reconciliation and the completion check, as long as it is inside the + // 24-hour stuck bound; without this test that bound could shrink far enough to break an ordinary wait jest.spyOn(scryptService, 'getWithdrawalStatus').mockResolvedValue({ id: 'w-inflight', status: ScryptTransactionStatus.COMPLETED, @@ -191,6 +202,18 @@ describe('ScryptAdapter', () => { await expect(adapter.checkCompletion(old)).resolves.toBe(false); }); + it('fails a withdrawal the venue has reported without a transaction hash for over 24 hours', async () => { + // without this bound, a withdrawal stuck at the venue with no txHash would poll forever with no exit + // but a human noticing — exactly the outcome the stuck-withdrawal backstop exists to remove + jest.spyOn(scryptService, 'getWithdrawalStatus').mockResolvedValue({ + id: 'w-stuck', + status: ScryptTransactionStatus.COMPLETED, + }); + const stuck = createWithdrawOrder({ created: new Date(Date.now() - 25 * 60 * 60 * 1000) }); + + await expect(adapter.checkCompletion(stuck)).rejects.toBeInstanceOf(OrderFailedException); + }); + it('still just waits while the withdrawal is young', async () => { jest.spyOn(scryptService, 'getWithdrawalStatus').mockResolvedValue(null); @@ -458,6 +481,240 @@ describe('ScryptAdapter', () => { }); }); + describe('cancelOutstanding', () => { + /** cancelOutstanding derives the trade pair from the rule, so the fixture needs one. */ + function cancellableOrder(overrides: Partial = {}): LiquidityManagementOrder { + return createUncertainSellOrder({ + action: { command: ScryptAdapterCommands.SELL, paramMap: { tradeAsset: 'USDT' } }, + pipeline: { rule: { targetAsset: { dexName: 'EUR' } } }, + ...overrides, + } as Partial); + } + + it('confirms only once the venue has settled every reference the order ever claimed', async () => { + const cancelIfOutstanding = jest + .spyOn(scryptService, 'cancelIfOutstanding') + .mockResolvedValue(ScryptCancellation.SETTLED); + const order = cancellableOrder(); + order.recordSpentCorrelationId('dfx-lm-4711-1'); + + await expect(adapter.cancelOutstanding(order)).resolves.toBe( + 'the venue answered for every reference that nothing is left to execute', + ); + expect(cancelIfOutstanding).toHaveBeenCalledTimes(2); + }); + + it('refuses when a single reference will not settle — that one could still spend the funds', async () => { + jest + .spyOn(scryptService, 'cancelIfOutstanding') + .mockImplementation(async (id: string) => + id === 'dfx-lm-4711' ? ScryptCancellation.UNCONFIRMED : ScryptCancellation.SETTLED, + ); + const order = cancellableOrder(); + order.recordSpentCorrelationId('dfx-lm-4711-1'); + + await expect(adapter.cancelOutstanding(order)).resolves.toBeNull(); + }); + + it('still cancels the older references when the newest one will not settle', async () => { + // leaving the loop at the first refusal would leave exactly those references without an attempt — + // and they are the ones that could be sitting open in the book + const cancelIfOutstanding = jest + .spyOn(scryptService, 'cancelIfOutstanding') + .mockImplementation(async (id: string) => + id === 'dfx-lm-4711-1' ? ScryptCancellation.UNCONFIRMED : ScryptCancellation.SETTLED, + ); + const order = cancellableOrder(); + order.recordSpentCorrelationId('dfx-lm-4711-1'); + + await expect(adapter.cancelOutstanding(order)).resolves.toBeNull(); + expect(cancelIfOutstanding).toHaveBeenCalledWith('dfx-lm-4711-1', 'EUR', 'USDT'); + expect(cancelIfOutstanding).toHaveBeenCalledWith('dfx-lm-4711', 'EUR', 'USDT'); + }); + + it('reconstructs a missing trade correlationId and still asks the venue under dfx-lm-${id}', async () => { + // invariant break stays logged; the reference is deterministic from the order id so a pure lookup is + // safe even when the column was never written + const cancelIfOutstanding = jest + .spyOn(scryptService, 'cancelIfOutstanding') + .mockResolvedValue(ScryptCancellation.SETTLED); + const errorSpy = jest.spyOn(adapter['logger'], 'error').mockImplementation(); + const order = cancellableOrder({ id: 4711, correlationId: null, previousCorrelationIds: null }); + + await expect(adapter.cancelOutstanding(order)).resolves.toBe( + 'the venue answered for every reference that nothing is left to execute', + ); + expect(errorSpy).toHaveBeenCalled(); + expect(cancelIfOutstanding).toHaveBeenCalledWith('dfx-lm-4711', 'EUR', 'USDT'); + }); + + it('treats an executed reference as settled — it cannot execute again either', async () => { + // holding on because something filled would be backwards: the fill is already in the venue's balance, + // and that balance is what the rule replans from, so it plans for what is actually left + jest + .spyOn(scryptService, 'cancelIfOutstanding') + .mockImplementation(async (id: string) => + id === 'dfx-lm-4711-1' ? ScryptCancellation.EXECUTED : ScryptCancellation.SETTLED, + ); + const order = cancellableOrder(); + order.recordSpentCorrelationId('dfx-lm-4711-1'); + + await expect(adapter.cancelOutstanding(order)).resolves.toBe( + 'the venue answered for every reference that nothing is left to execute', + ); + // an abandoned order books no output, so the reference that filled has to be named somewhere + expect(order.errorMessage).toContain('dfx-lm-4711-1'); + }); + + it('does not cancel a withdrawal via cancelIfOutstanding — absence is confirmed instead', async () => { + const cancelIfOutstanding = jest.spyOn(scryptService, 'cancelIfOutstanding'); + jest.spyOn(scryptService, 'confirmWithdrawalAbsent').mockResolvedValue(false); + const order = cancellableOrder({ action: withdrawAction() }); + + await expect(adapter.cancelOutstanding(order)).resolves.toBeNull(); + expect(cancelIfOutstanding).not.toHaveBeenCalled(); + expect(scryptService.confirmWithdrawalAbsent).toHaveBeenCalledWith(order.correlationId); + }); + + it('abandons a withdrawal once the venue answers without naming it', async () => { + const cancelIfOutstanding = jest.spyOn(scryptService, 'cancelIfOutstanding'); + jest.spyOn(scryptService, 'confirmWithdrawalAbsent').mockResolvedValue(true); + const order = cancellableOrder({ action: withdrawAction() }); + + await expect(adapter.cancelOutstanding(order)).resolves.toBe( + 'the venue answered with its transaction history and did not name this withdrawal', + ); + expect(cancelIfOutstanding).not.toHaveBeenCalled(); + expect(scryptService.confirmWithdrawalAbsent).toHaveBeenCalledWith(order.correlationId); + }); + + it('reconstructs a missing withdrawal correlationId and still confirms absence under dfx-lm-${id}', async () => { + const confirmWithdrawalAbsent = jest.spyOn(scryptService, 'confirmWithdrawalAbsent').mockResolvedValue(true); + const errorSpy = jest.spyOn(adapter['logger'], 'error').mockImplementation(); + const order = cancellableOrder({ id: 4711, action: withdrawAction(), correlationId: null }); + + await expect(adapter.cancelOutstanding(order)).resolves.toBe( + 'the venue answered with its transaction history and did not name this withdrawal', + ); + expect(errorSpy).toHaveBeenCalled(); + expect(confirmWithdrawalAbsent).toHaveBeenCalledWith('dfx-lm-4711'); + }); + + it('for an unsupported command asks getOrderStatus and returns a reason when every reference is already terminal', async () => { + // The tradeAsset-derived shortcut is gone (see the comment above the branch in cancelOutstanding): a + // legacy/renamed command always asks the venue first, whether or not paramMap still carries tradeAsset. + const getOrderStatus = jest + .spyOn(scryptService, 'getOrderStatus') + .mockImplementation(async (id: string) => venueOrder(id, ScryptOrderStatus.FILLED)); + const cancelIfOutstandingBySymbol = jest.spyOn(scryptService, 'cancelIfOutstandingBySymbol'); + const order = cancellableOrder({ + action: sellIfDeficitAction({}), + }); + order.recordSpentCorrelationId('dfx-lm-4711-1'); + + await expect(adapter.cancelOutstanding(order)).resolves.toBe( + 'the venue left no reference of this unsupported command able to execute — each is terminal or unknown to it', + ); + expect(cancelIfOutstandingBySymbol).not.toHaveBeenCalled(); + expect(getOrderStatus).toHaveBeenCalledWith('dfx-lm-4711-1'); + expect(getOrderStatus).toHaveBeenCalledWith('dfx-lm-4711'); + }); + + it('for an unsupported command treats venue-unknown (null) as settled and returns a reason', async () => { + // null = venue answered and has no record for that reference — same inference as refusedAsUnknown / + // SCRYPT_UNKNOWN_ORDER on the active path; must not keep the order quarantined. + const getOrderStatus = jest + .spyOn(scryptService, 'getOrderStatus') + .mockImplementation(async (id: string) => + id === 'dfx-lm-4711-1' ? null : venueOrder(id, ScryptOrderStatus.FILLED), + ); + const cancelIfOutstandingBySymbol = jest.spyOn(scryptService, 'cancelIfOutstandingBySymbol'); + const order = cancellableOrder({ + action: sellIfDeficitAction({}), + }); + order.recordSpentCorrelationId('dfx-lm-4711-1'); + + await expect(adapter.cancelOutstanding(order)).resolves.toBe( + 'the venue left no reference of this unsupported command able to execute — each is terminal or unknown to it', + ); + expect(cancelIfOutstandingBySymbol).not.toHaveBeenCalled(); + expect(getOrderStatus).toHaveBeenCalledWith('dfx-lm-4711-1'); + expect(getOrderStatus).toHaveBeenCalledWith('dfx-lm-4711'); + }); + + it('for an unsupported command cancels a non-terminal reference under the symbol the venue supplied, and returns a reason once it settles', async () => { + // FIX B: the venue's own getOrderStatus reply names the symbol, so a non-terminal legacy reference is + // no longer just polled — it is actively cancelled under exactly that symbol, no tradeAsset needed. + jest + .spyOn(scryptService, 'getOrderStatus') + .mockImplementation(async (id: string) => + id === 'dfx-lm-4711-1' + ? venueOrder(id, ScryptOrderStatus.NEW, 'XRP/USDT') + : venueOrder(id, ScryptOrderStatus.FILLED), + ); + const cancelIfOutstandingBySymbol = jest + .spyOn(scryptService, 'cancelIfOutstandingBySymbol') + .mockResolvedValue(ScryptCancellation.SETTLED); + const order = cancellableOrder({ + action: sellIfDeficitAction({}), + }); + order.recordSpentCorrelationId('dfx-lm-4711-1'); + + await expect(adapter.cancelOutstanding(order)).resolves.toBe( + 'the venue left no reference of this unsupported command able to execute — each is terminal or unknown to it', + ); + expect(cancelIfOutstandingBySymbol).toHaveBeenCalledWith('dfx-lm-4711-1', 'XRP/USDT'); + }); + + it('for an unsupported command records a reference the symbol-based cancel reports as executed', async () => { + // Mirrors the known-command path: a fill is not a reason to hold on, but it is worth naming so it can + // be reconciled against the venue balance. + jest + .spyOn(scryptService, 'getOrderStatus') + .mockImplementation(async (id: string) => venueOrder(id, ScryptOrderStatus.PARTIALLY_FILLED, 'XRP/USDT')); + jest.spyOn(scryptService, 'cancelIfOutstandingBySymbol').mockResolvedValue(ScryptCancellation.EXECUTED); + const order = cancellableOrder({ + action: sellIfDeficitAction({}), + }); + + await adapter.cancelOutstanding(order); + + expect(order.errorMessage).toContain('dfx-lm-4711'); + }); + + it('for an unsupported command keeps the order quarantined when the symbol-based cancel stays UNCONFIRMED', async () => { + jest + .spyOn(scryptService, 'getOrderStatus') + .mockImplementation(async (id: string) => venueOrder(id, ScryptOrderStatus.NEW, 'XRP/USDT')); + const cancelIfOutstandingBySymbol = jest + .spyOn(scryptService, 'cancelIfOutstandingBySymbol') + .mockResolvedValue(ScryptCancellation.UNCONFIRMED); + const order = cancellableOrder({ + action: sellIfDeficitAction({}), + }); + order.recordSpentCorrelationId('dfx-lm-4711-1'); + + await expect(adapter.cancelOutstanding(order)).resolves.toBeNull(); + expect(cancelIfOutstandingBySymbol).toHaveBeenCalled(); + }); + + it('for an unsupported command returns null when any reference is unreachable, without attempting a cancel', async () => { + // undefined (fetch failure / catch) is not a venue answer — keep waiting; must stay distinct from null. + jest.spyOn(scryptService, 'getOrderStatus').mockImplementation(async (id: string) => { + if (id === 'dfx-lm-4711-1') throw new Error('Connection closed'); + return venueOrder(id, ScryptOrderStatus.FILLED); + }); + const cancelIfOutstandingBySymbol = jest.spyOn(scryptService, 'cancelIfOutstandingBySymbol'); + const order = cancellableOrder({ + action: sellIfDeficitAction({}), + }); + order.recordSpentCorrelationId('dfx-lm-4711-1'); + + await expect(adapter.cancelOutstanding(order)).resolves.toBeNull(); + expect(cancelIfOutstandingBySymbol).not.toHaveBeenCalled(); + }); + }); + describe('resolveUncertainOrder', () => { it('reports SENT when the venue knows the reference', async () => { jest.spyOn(scryptService, 'getOrderStatus').mockResolvedValue(venueOrder('dfx-lm-4711')); @@ -476,6 +733,21 @@ describe('ScryptAdapter', () => { await expect(adapter.resolveUncertainOrder(ancient)).resolves.toBe(UncertainOrderResolution.UNRESOLVED); }); + it.each([[undefined], [null]])( + 'reconstructs a missing reference %p as dfx-lm-${id} and still asks the venue', + async (correlationId: string | null | undefined) => { + // invariant break stays logged; the reconstructed id is deterministic from the order row so a pure + // lookup is safe, and leaving the order for an operator forever is not allowed + const getOrderStatus = jest.spyOn(scryptService, 'getOrderStatus').mockResolvedValue(null); + const errorSpy = jest.spyOn(adapter['logger'], 'error').mockImplementation(); + const order = createUncertainSellOrder({ id: 4711, correlationId, previousCorrelationIds: null }); + + await expect(adapter.resolveUncertainOrder(order)).resolves.toBe(UncertainOrderResolution.UNRESOLVED); + expect(errorSpy).toHaveBeenCalled(); + expect(getOrderStatus).toHaveBeenCalledWith('dfx-lm-4711', expect.any(Date)); + }, + ); + it('reports UNAVAILABLE when the lookup itself fails — no question reached the venue', async () => { jest.spyOn(scryptService, 'getOrderStatus').mockRejectedValue(new Error('Connection closed')); @@ -484,12 +756,6 @@ describe('ScryptAdapter', () => { ); }); - it('stays UNRESOLVED when no reference was ever reserved', async () => { - const order = createUncertainSellOrder({ correlationId: undefined }); - - await expect(adapter.resolveUncertainOrder(order)).resolves.toBe(UncertainOrderResolution.UNRESOLVED); - }); - it('uses the withdrawal lookup for withdraw orders', async () => { jest.spyOn(scryptService, 'findWithdrawal').mockResolvedValue(null); const order = createUncertainSellOrder({ @@ -550,7 +816,8 @@ describe('ScryptAdapter', () => { it('stays quarantined when a claimed replacement is not (yet) visible, even if the predecessor is', async () => { // an accepted replacement may lag in the venue's view; falling back to the order it replaced would - // report SENT on a superseded reference and leave the live replacement untracked + // report SENT on a superseded reference and leave the live replacement untracked. + // jest .spyOn(scryptService, 'getOrderStatus') .mockImplementation(async (id: string) => (id === 'dfx-lm-4711' ? venueOrder(id) : null)); @@ -560,6 +827,16 @@ describe('ScryptAdapter', () => { await expect(adapter.resolveUncertainOrder(order)).resolves.toBe(UncertainOrderResolution.UNRESOLVED); }); + it('reports a single unknown reference as UNRESOLVED — nothing was left unasked', async () => { + // the ordinary case: one attempt, the venue answered about it, and there is no older reference the + // lookup skipped. That is a complete answer, and the caller's bound may act on it. + jest.spyOn(scryptService, 'getOrderStatus').mockResolvedValue(null); + + await expect(adapter.resolveUncertainOrder(createUncertainSellOrder())).resolves.toBe( + UncertainOrderResolution.UNRESOLVED, + ); + }); + it('falls back to the predecessor only after the replacement was explicitly rejected', async () => { jest .spyOn(scryptService, 'getOrderStatus') @@ -608,5 +885,14 @@ describe('ScryptAdapter', () => { expect(since?.getTime()).toBeGreaterThanOrEqual(earliest.getTime()); expect(since?.getTime()).toBeLessThanOrEqual(latest.getTime()); }); + + it('resolves an unknown command on the command-independent trade path', async () => { + jest.spyOn(scryptService, 'getOrderStatus').mockResolvedValue(venueOrder('dfx-lm-4711')); + const order = createUncertainSellOrder({ + action: sellIfDeficitAction(), + }); + + await expect(adapter.resolveUncertainOrder(order)).resolves.toBe(UncertainOrderResolution.SENT); + }); }); }); diff --git a/src/subdomains/core/liquidity-management/adapters/actions/scrypt.adapter.ts b/src/subdomains/core/liquidity-management/adapters/actions/scrypt.adapter.ts index 2d4f2c155d..27723d8e04 100644 --- a/src/subdomains/core/liquidity-management/adapters/actions/scrypt.adapter.ts +++ b/src/subdomains/core/liquidity-management/adapters/actions/scrypt.adapter.ts @@ -1,6 +1,8 @@ import { Injectable } from '@nestjs/common'; import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.enum'; import { + isTerminalScryptOrderStatus, + ScryptCancellation, ScryptOrderInfo, ScryptOrderSide, ScryptOrderStatus, @@ -11,6 +13,7 @@ import { isVenueRejection, ScryptAmendRejectedError, ScryptOrderNotFoundError, + ScryptOrderStuckPendingError, ScryptUnconfirmedWriteError, } from 'src/integration/exchange/services/scrypt-websocket-connection'; import { ScryptService } from 'src/integration/exchange/services/scrypt.service'; @@ -41,11 +44,41 @@ const SCRYPT_CORRELATION_PREFIX = 'dfx-lm-'; /** * How long an acknowledged order may stay unobservable before it is quarantined rather than polled again. * - * Matches the age at which the venue lookup itself gives up on finding an order, so both routes out of a - * silent order agree. Quarantine is not a verdict — the order is still not declared failed — it only moves it - * somewhere a human can act on. + * Five minutes, matching the age at which the venue lookup itself gives up on finding an order + * (`ORDER_LOST_AFTER_MINUTES`), so both routes out of a silent order still agree. Together with the + * five-minute abandon bound for these commands that is the ten-minute ceiling the orderer set — nominal, + * and conditional on the venue answering at all: the pass runs on a ten-second cron with jitter, and a + * venue that cannot be reached holds the order past that point, because the exit rests on its answer + * rather than on the clock. See ABANDON_UNCERTAIN_MINUTES.VENUE_WITHDRAWAL for the same bound argued + * from the other end. A venue record is written when a request is ACCEPTED, not when it finishes — so its + * absence after five minutes says the acceptance is in doubt, which is independent of a withdrawal itself + * being allowed to take hours. Quarantine is not a verdict — the order is still not declared failed here — it only + * moves it where the caller's bound can attempt an automatic exit (cancel every trade reference, or confirm + * a withdrawal is unnamed in the venue's history reply). An operator can still release sooner as a shortcut; the human + * is not the rule path for either command. */ -const SCRYPT_UNOBSERVABLE_QUARANTINE_MINUTES = 60; +const SCRYPT_UNOBSERVABLE_QUARANTINE_MINUTES = 5; + +/** + * Backstop against a withdrawal whose venue record never gains a transaction hash — not a bound against a + * merely slow one. + * + * `ScryptTransactionStatus` (scrypt.dto.ts) declares only COMPLETED | FAILED | REJECTED, and FAILED/REJECTED + * are already handled above this branch, so what actually reaches here is either a Completed record without + * a hash or a status outside that declared enum — not a documented "in progress" state either way. What is + * certain is only the absence: a record without a `txHash` counts as not finished, whatever status the venue + * attaches to it. Without a ceiling here the only exit left would be a human noticing, and that is exactly + * the outcome this system exists to remove. Measured over 60 days in production, withdrawals took a median + * of 6.6 minutes to complete, a p95 of 90 minutes, and the single slowest observed run took 336 minutes (5.6 + * hours). 24 hours is a good four times that worst case — far outside anything slowness alone could explain. + * + * The exit is failing the order so the rule can replan; if the venue pays out afterwards regardless, that + * is only an internal rebooking, because every Scrypt withdrawal address is DFX's own. Deliberately NOT + * folded into the quarantine bound above: an order with a venue record already on file makes the absence + * proof `confirmWithdrawalAbsent` relies on unreachable, and the order would only bounce between + * reconciliation and this completion check instead of ever resolving. + */ +const SCRYPT_WITHDRAWAL_STUCK_AFTER_MINUTES = 24 * 60; @Injectable() export class ScryptAdapter extends LiquidityActionAdapter { @@ -230,12 +263,22 @@ export class ScryptAdapter extends LiquidityActionAdapter { // No record at all, past the age at which the venue is considered to have lost it: we cannot tell // whether this withdrawal happened, and the manual path only accepts quarantined orders, so leaving it // here would mean no way out at all. A record WITHOUT a hash is different — that is an observation, the - // withdrawal is simply still in flight, and quarantining it would only bounce it back and forth. + // withdrawal is simply still in flight, but only up to SCRYPT_WITHDRAWAL_STUCK_AFTER_MINUTES: short of + // that bound quarantining it would only bounce it back and forth between reconciliation and this check. if (!withdrawal && Util.minutesDiff(order.created) > SCRYPT_UNOBSERVABLE_QUARANTINE_MINUTES) throw new OrderOutcomeUnknownException( `Scrypt has no record of withdrawal ${correlationId} after more than ${SCRYPT_UNOBSERVABLE_QUARANTINE_MINUTES} minutes`, ); + if (withdrawal) { + const ageMinutes = Util.minutesDiff(order.created); + if (ageMinutes > SCRYPT_WITHDRAWAL_STUCK_AFTER_MINUTES) + throw new OrderFailedException( + `Withdrawal ${correlationId} is ${Math.round(ageMinutes)} minutes old: the venue reports status ` + + `${withdrawal.status} without a transaction hash, longer than the ${SCRYPT_WITHDRAWAL_STUCK_AFTER_MINUTES}-minute stuck bound`, + ); + } + this.logger.verbose(`No withdrawal id for id ${correlationId} at ${this.scryptService.name} found`); return false; } @@ -260,6 +303,192 @@ export class ScryptAdapter extends LiquidityActionAdapter { return this.checkTradeCompletion(order, tradeAsset, asset); } + /** + * Make sure nothing this order could still have live can execute, so the caller may give it up. + * + * For trades: every reference the row ever claimed — sent or merely reserved — not just the current one. + * The reason an order gets here is that at least one of them has an outcome nobody could observe, and an + * unobserved reference is precisely the one that might be sitting in the book. These are GTC orders — + * nothing expires them — so age is no argument at all, and the only way to know a reference cannot fill is + * to have the venue say so. All-or-nothing on purpose: one reference the venue would not settle is enough + * to keep the whole order quarantined, because the funds a rule would get back are the same funds that + * reference could still spend. Every reference is still asked about — a refusal on one is no reason to + * leave the others without an attempt. An unsupported/legacy command cancels the same way; it just names + * its cancel symbol from the venue's own order-status reply instead of deriving one locally, so there is + * no command for which "no symbol" blocks this exit. + * + * For withdrawals: Scrypt has no cancel operation. The exit rests on absence — the venue answered + * successfully and that reply has no record of this reference. Deliberately no completeness check on that + * reply: withdrawal destinations are DFX-owned, so a truncated answer costs an internal rebooking, while a + * check demanding local anchors would strand the order forever (see `confirmWithdrawalAbsent`). So this is + * weaker than "the request never arrived" AND weaker than "the history was complete" — it is only "the + * venue answered and did not name it". Exactly three answers return null and leave the order quarantined + * for another pass: a lookup that failed, a reply that does name the reference, or the reference surfacing + * in the live cache while the lookup was open. An empty reply is none of them — a reference cannot be in a + * history with no rows, so that confirms absence. + * + * Returns the reason string the caller records on abandon, or null when nothing is settled yet. + * + * The replan that follows reads the venue's balance, which is pushed rather than polled, so a fill that + * has only just landed may not be in it yet. In practice that push follows a fill within seconds and the + * abandoned order is the slower half of the race, but it is a window rather than a guarantee — worth + * knowing if a rule is ever seen planning against a balance that looks one fill stale. + */ + async cancelOutstanding(order: LiquidityManagementOrder): Promise { + // Reconstruct a missing reference from the order id alone (see reserveCorrelationId). The error log + // remains: reserve-before-send should make this state impossible. Reconstruction is still safe to ask + // about — the id is deterministic from the row, has no random component, and a pure venue lookup under + // it is harmless if nothing was ever sent under that name. + if (!order.correlationId) { + this.logger.error( + `Order ${order.id}: Scrypt order reached cancelOutstanding with no correlationId — reserve-before-send should make this impossible`, + ); + order.reserveCorrelationId(this.reserveCorrelationId(order)); + } + + if (order.action.command === ScryptAdapterCommands.WITHDRAW) { + const absent = await this.scryptService.confirmWithdrawalAbsent(order.correlationId); + return absent ? 'the venue answered with its transaction history and did not name this withdrawal' : null; + } + + const references = this.attemptedReferencesNewestFirst(order); + + // Nothing ever went out under a reference, so the venue cannot confirm anything about this order — and + // an empty loop would otherwise fall through to "all settled" without a single question asked. Absent + // evidence this must hold the order, never release it. Since reserve-before-send a Scrypt order should + // not reach quarantine without a reference; this is an invariant break, not a planned wait. + if (!references.length) { + this.logger.error( + `Order ${order.id}: Scrypt trade reached cancelOutstanding with no references — reserve-before-send should make this impossible`, + ); + return null; + } + + // A command no longer in ScryptAdapterCommands (rename/removal) still reaches this adapter via + // getReconciliationIntegration. There used to be a second path here for such a command when its + // paramMap still carried a `tradeAsset` — but that was only ever a shortcut to a symbol, and a shortcut + // that could be wrong: a `tradeAsset` in a stale paramMap is not a guarantee the command was ever a + // plain sell/buy, "SELL vs. everything else" is not a real side inference for a command that, by + // definition, is not SELL, and getTradePair reads live security configuration that a delisted or + // renamed pair could no longer match even though the order itself is still open at the venue under its + // original symbol. None of that guessing is needed: the venue's own order-status reply already names + // the symbol a reference lives under (ScryptOrderInfo.symbol), so every reference the venue can still + // show us is cancellable through the symbol it hands back, with no local trade-pair derivation at all. + // One way for every unsupported command, not two: ask first, storno under the venue's own symbol only + // if the answer is non-terminal. `null` vs `undefined` is the whole point here (same distinction as + // adoptLiveReplacement): null = venue answered and has no record; undefined = could not be asked. + const knownCommands = Object.values(ScryptAdapterCommands) as string[]; + const isKnownCommand = knownCommands.includes(order.action.command); + + if (!isKnownCommand) { + const executed: CorrelationId[] = []; + let unsettled = 0; + + for (const reference of references) { + const info = await this.scryptService.getOrderStatus(reference).catch(() => undefined); + + if (info === undefined) { + this.logger.warn( + `Order ${order.id}: unsupported command ${order.action.command} — reference ${reference} is unreachable; keeping the order quarantined`, + ); + return null; + } + + if (info === null) { + // Venue answered: no record for this reference. Same inference as cancelIfOutstanding / + // refusedAsUnknown (SCRYPT_UNKNOWN_ORDER): nothing left that can execute under it. + this.logger.info( + `Order ${order.id}: unsupported command ${order.action.command} — reference ${reference} is unknown to the venue — treated as settled`, + ); + continue; + } + + if (isTerminalScryptOrderStatus(info.status)) continue; + + // Non-terminal, and the venue just named the symbol it lives under in the very same reply — storno + // it under exactly that symbol. Evaluated the same way the active path below evaluates a cancel: + // SETTLED moves on to the next reference, EXECUTED is recorded so the fill can be reconciled, + // anything else keeps the whole order quarantined (a single unsettled reference could still spend + // the funds). + const outcome = await this.scryptService.cancelIfOutstandingBySymbol(reference, info.symbol); + + if (outcome === ScryptCancellation.SETTLED) continue; + + if (outcome === ScryptCancellation.EXECUTED) { + executed.push(reference); + continue; + } + + unsettled++; + this.logger.warn( + `Order ${order.id}: unsupported command ${order.action.command} — Scrypt would not settle ${reference}, so it may still execute — keeping the order quarantined`, + ); + } + + if (unsettled) return null; + + if (executed.length) + order.errorMessage = `${order.errorMessage} (executed at Scrypt under ${executed.join(', ')})`; + + this.logger.info( + `Order ${order.id}: venue leaves no reference of unsupported command ${order.action.command} able to execute — each is terminal or unknown to it${ + executed.length ? `; ${executed.join(', ')} had filled` : '' + }`, + ); + return 'the venue left no reference of this unsupported command able to execute — each is terminal or unknown to it'; + } + + // Known command: from/to is derived directly from the rule, so the venue's cancel-and-report is asked + // for every reference straight away — no separate status lookup needed first (contrast the branch + // above, which cannot derive a trade pair locally and asks first for that reason). + const { tradeAsset } = this.parseTradeParams(order.action.paramMap); + const asset = order.pipeline.rule.targetAsset.dexName; + const [from, to] = order.action.command === ScryptAdapterCommands.SELL ? [asset, tradeAsset] : [tradeAsset, asset]; + + const executed: CorrelationId[] = []; + let unsettled = 0; + + for (const reference of references) { + const outcome = await this.scryptService.cancelIfOutstanding(reference, from, to); + + // Cancelled and executed both answer the only question that matters here: can this reference still + // execute? It cannot — one because it was called off, the other because it ran to a terminal state. + // A fill is not a reason to hold on: it is already in the venue's balance, and that balance is what + // the rule replans from, so it plans for what is actually left rather than for what this row + // believed. Which reference filled is recorded on the order below, since the row itself no longer + // carries that after being abandoned. + if (outcome === ScryptCancellation.SETTLED) continue; + + if (outcome === ScryptCancellation.EXECUTED) { + executed.push(reference); + continue; + } + + // Counted, not returned on. Leaving the loop here would leave every older reference without so much + // as a cancellation attempt — and those are exactly the ones that can sit open in the book while the + // newest keeps refusing to settle. Ask about all of them, then decide. + unsettled++; + this.logger.warn( + `Order ${order.id}: Scrypt would not settle ${reference}, so it may still execute — keeping the order quarantined`, + ); + } + + if (unsettled) return null; + + // Which reference filled is the one thing an abandoned order can no longer say for itself — its status + // becomes FAILED and it books no output. The venue's own transaction record carries the money side, but + // tying that back to this row afterwards needs the reference named somewhere, so name it here. + if (executed.length) order.errorMessage = `${order.errorMessage} (executed at Scrypt under ${executed.join(', ')})`; + + this.logger.info( + `Order ${order.id}: Scrypt answered for every reference that nothing is left to execute${ + executed.length ? `; ${executed.join(', ')} had filled` : '' + }`, + ); + + return 'the venue answered for every reference that nothing is left to execute'; + } + private async checkTradeCompletion(order: LiquidityManagementOrder, from: string, to: string): Promise { // Before anything may write again: a previous pass may have had its replacement accepted and then failed // to record it, leaving this row pointing at the predecessor the venue has already cancelled. Restarting @@ -318,9 +547,19 @@ export class ScryptAdapter extends LiquidityActionAdapter { } // The venue once acknowledged this order and now cannot find it. That is not a failure — it may have - // filled or been cancelled outside our view — so it goes to a human instead of releasing the rule. + // filled or been cancelled outside our view — so it goes to quarantine instead of releasing the rule. + // From there the caller's bound attempts a cancellation whose confirmation ends it; an operator can + // still release sooner as a shortcut, but is not the rule path. if (e instanceof ScryptOrderNotFoundError) throw new OrderOutcomeUnknownException(e.message); + // Not a blind spot: asked to cancel a reference past its bound, the venue settled it. See + // {@link ScryptOrderStuckPendingError} for what that settlement rests on: a terminal cancel with + // nothing filled, or the venue no longer recognising the reference — the latter an inference from + // SCRYPT_UNKNOWN_ORDER rather than a directly observed fact. Either reading lands at the same + // conclusion, so the order fails outright and the rule may replan straight away instead of waiting on + // a bound already spent. + if (e instanceof ScryptOrderStuckPendingError) throw new OrderFailedException(e.message); + // A rejection is a reply: the venue reached a verdict, so the order really did end. if (isVenueRejection(e)) throw new OrderFailedException(e.message); @@ -355,6 +594,11 @@ export class ScryptAdapter extends LiquidityActionAdapter { * is a barrier rather than something to step past — the reference is recorded BEFORE the request leaves, * so one the venue does not show may still be live there, and carrying on with the predecessor would put a * second request next to it. + * + * The barrier is meant to hold. What eventually ends such an order is not this method giving way, but the + * caller cancelling every reference it ever claimed — once the venue answers that none of them can + * execute, the + * claim is settled and there is nothing left to block on. */ private async adoptLiveReplacement(order: LiquidityManagementOrder): Promise { const currentAttempt = this.attemptNumber(order, order.correlationId); @@ -398,8 +642,9 @@ export class ScryptAdapter extends LiquidityActionAdapter { * Waiting is the safe answer — writing against an order whose true state is unknown is how a second * request against the same funds happens. But not forever: the manual path only accepts quarantined * orders, so an order nobody can ever observe would poll for good with no way out at all. Past the same - * age at which the venue itself is considered to have lost an order, it goes to a human instead — still - * not declared failed. + * age at which the venue itself is considered to have lost an order, it goes to quarantine instead — + * still not declared failed here. From there the caller's bound ends it via cancellation confirmation; + * an operator can still release sooner as a shortcut. */ private waitOrQuarantine(order: LiquidityManagementOrder, reason: string): boolean { if (Util.minutesDiff(order.created) > SCRYPT_UNOBSERVABLE_QUARANTINE_MINUTES) @@ -413,12 +658,13 @@ export class ScryptAdapter extends LiquidityActionAdapter { } /** - * Every reference this order has actually put on the wire, newest first. + * Every reference this order has claimed — sent or merely reserved — newest first. * * Ordered by the attempt suffix rather than by storage order, so it does not depend on how the list was * assembled. Deliberately does NOT include the next reference: that one has not been sent, and looking for - * it would stop the search on an absence that means nothing — leaving the reference that WAS sent unchecked - * and the order quarantined for good. + * it would stop the search on an absence that means nothing — leaving the reference that WAS sent + * unchecked, and the order quarantined until a cancellation the caller's bound attempts settles a request + * that may well be live. */ private attemptedReferencesNewestFirst(order: LiquidityManagementOrder): CorrelationId[] { return [...order.allCorrelationIds].sort((a, b) => this.attemptNumber(order, b) - this.attemptNumber(order, a)); @@ -582,21 +828,37 @@ export class ScryptAdapter extends LiquidityActionAdapter { // follows rejects the pending request with a generic message that says nothing about whether the venue // acted on them. Both become unknown outcomes. // - // Over-classifying costs an operator a look at the venue; under-classifying is what moved money without - // a record. Since absence at the venue is not proof, such an order waits for a human rather than - // resolving itself — deliberately the expensive direction, because the cheap one is the dangerous one. + // Over-classifying costs another automatic pass against the venue; under-classifying is what moved money + // without a record. Since absence at the venue is not proof, such an order waits on the venue — for a + // cancellation confirmation (trade) or a history reply that does not name it (withdraw) once its bound + // is reached — rather than resolving itself here, deliberately the expensive direction, because the cheap + // one is the dangerous one. An operator can still release sooner as a shortcut. return new OrderOutcomeUnknownException(`Scrypt gave no confirmed outcome for the ${description}: ${e.message}`); } /** * Ask Scrypt what happened to a quarantined order. Observes only — never re-sends. * - * Can only ever confirm a positive: Scrypt has no terminal "this reference was never accepted" reply, so - * a missing record leaves the order quarantined for a human rather than releasing its rule. + * Only a matched reference can confirm a positive. A missing record confirms nothing on its own — Scrypt + * has no terminal "this reference was never accepted" reply — so it leaves the order quarantined. What + * ends it is not this method concluding anything, but the caller settling every reference the order ever + * claimed: cancel confirmation for trades, or an unnaming history reply for withdrawals. Once + * the venue answers that nothing can still execute, giving up is a fact rather than a guess. An explicit + * rejection of every attempted trade reference is the one negative that settles here, and returns NOT_SENT. */ async resolveUncertainOrder(order: LiquidityManagementOrder): Promise { + // Reconstruct a missing reference from the order id alone (see reserveCorrelationId). The error log + // remains: reserve-before-send should make this state impossible. Reconstruction is still safe — the id + // is deterministic from the row, has no random component, and a pure venue lookup under it is harmless + // if nothing was ever sent under that name. Without it the order would wait on an operator forever. + if (!order.correlationId) { + this.logger.error( + `Order ${order.id}: Scrypt order reached resolveUncertainOrder with no correlationId — reserve-before-send should make quarantining without a reference impossible; this is a bug`, + ); + order.reserveCorrelationId(this.reserveCorrelationId(order)); + } + const { correlationId } = order; - if (!correlationId) return UncertainOrderResolution.UNRESOLVED; let allAttemptsRejected = false; @@ -608,6 +870,11 @@ export class ScryptAdapter extends LiquidityActionAdapter { return UncertainOrderResolution.SENT; } } else { + // Trade path (and any command that is not WITHDRAW, including unknown/renamed commands): works for + // every command because it never consults order.action.command or parseTradeParams — only + // attemptedReferencesNewestFirst and getOrderStatus. The outer branch is `command === WITHDRAW`, + // which is false for an unknown command, so those fall here correctly and stay command-independent. + // // Newest first. A replacement supersedes the order it replaced, and the replaced one usually still // exists at the venue in a cancelled state — checking oldest first would match that, report SENT and // leave the live replacement untracked while the completion check polls a superseded reference. @@ -629,11 +896,12 @@ export class ScryptAdapter extends LiquidityActionAdapter { this.logger.warn( `Scrypt does not (yet) know reference ${candidate} for order ${order.id} — keeping it quarantined`, ); + return UncertainOrderResolution.UNRESOLVED; } - // A refused replacement never took effect and leaves its predecessor live. This is the only case in - // which an older reference may be considered. + // A refused replacement never took effect and leaves its predecessor live. This is the only case + // with an explicit reply that reaches an older reference — the timeout above is the other way. if (info.status === ScryptOrderStatus.REJECTED) { order.recordSpentCorrelationId(candidate); rejectedCount++; @@ -660,8 +928,14 @@ export class ScryptAdapter extends LiquidityActionAdapter { // Absence is NOT proof. A snapshot without the reference may simply predate the venue registering it, // and Scrypt offers no terminal "this was never accepted" acknowledgement to rely on. Concluding - // otherwise is what would let the rule reissue a request that later materialises — so the order stays - // quarantined for a human, and the rule stays blocked, which is the safe direction. + // otherwise is what would let the rule reissue a request that later materialises — so this reports + // only what it saw, and never resolves the order on absence alone. + // + // The caller bounds the wait: an order stuck here long enough gets an automatic exit attempt (cancel + // every trade reference, or a history reply that does not name the withdrawal) rather than being held for + // an operator who may never come, and only that attempt's confirmation abandons it. Both belong there, + // not here — this method's job is to report what the venue said, not to decide how long a rule may stay + // blocked or when giving up is safe. this.logger.warn( `Scrypt still has no record of reference ${correlationId} for order ${order.id} — keeping it quarantined`, ); diff --git a/src/subdomains/core/liquidity-management/entities/__tests__/liquidity-management-order.entity.spec.ts b/src/subdomains/core/liquidity-management/entities/__tests__/liquidity-management-order.entity.spec.ts index 3e7995c837..c4fbcefe48 100644 --- a/src/subdomains/core/liquidity-management/entities/__tests__/liquidity-management-order.entity.spec.ts +++ b/src/subdomains/core/liquidity-management/entities/__tests__/liquidity-management-order.entity.spec.ts @@ -26,6 +26,180 @@ describe('LiquidityManagementOrder', () => { }); }); + describe('unresolvableTooLong', () => { + function quarantined(minutes?: number, command = 'sell', system = 'Scrypt'): LiquidityManagementOrder { + return Object.assign(new LiquidityManagementOrder(), { + status: LiquidityManagementOrderStatus.UNCERTAIN, + updated: minutes == null ? undefined : minutesAgo(minutes), + action: { system, command }, + }); + } + + it('is false without a quarantine timestamp — a missing date reads as the epoch, which would fire at once', () => { + expect(quarantined(undefined).unresolvableTooLong()).toBe(false); + }); + + it.each(['sell', 'buy'])('applies the short trade bound to scrypt/%s', (command) => { + expect(quarantined(4, command).unresolvableTooLong()).toBe(false); + expect(quarantined(6, command).unresolvableTooLong()).toBe(true); + }); + + it('applies the short bound to scrypt/withdraw — the venue answers about its own history', () => { + // Not derived from how long withdrawals take (they reach hours): the bound only applies while the + // venue does not name the reference, and it exists to hold the ten-minute ceiling together with the + // unobservable window. + expect(quarantined(4, 'withdraw').unresolvableTooLong()).toBe(false); + expect(quarantined(6, 'withdraw').unresolvableTooLong()).toBe(true); + }); + + it('leaves the long bound on a transfer at a venue that cannot be asked', () => { + // Lowering the shared transfer bound would have reached bridges and mints too, whose adapters cannot + // attempt any exit — they would only be polled more often for nothing. + expect(quarantined(11 * 60, 'withdraw', 'Kraken').unresolvableTooLong()).toBe(false); + expect(quarantined(13 * 60, 'withdraw', 'Kraken').unresolvableTooLong()).toBe(true); + }); + + it('applies the long bound to a command that only looks like a trade elsewhere', () => { + // an on-chain swap is not a book match, whatever it is called + expect(quarantined(30, 'sell', 'DfxDex').unresolvableTooLong()).toBe(false); + }); + + it('applies the long bound to anything unrecognised', () => { + expect(quarantined(30, 'some-new-command', 'SomeNewSystem').unresolvableTooLong()).toBe(false); + }); + + it('falls back to created when updated is missing — bound can still expire', () => { + // deliberate fallback: created is older-or-equal, so the bound fires earlier, not later; abandon still + // needs venue confirmation, so an earlier cleanup attempt is safe + const order = Object.assign(new LiquidityManagementOrder(), { + status: LiquidityManagementOrderStatus.UNCERTAIN, + updated: undefined, + created: minutesAgo(6), + action: { system: 'Scrypt', command: 'sell' }, + }); + expect(order.unresolvableTooLong()).toBe(true); + }); + + it('stays false when both updated and created are missing', () => { + const order = Object.assign(new LiquidityManagementOrder(), { + status: LiquidityManagementOrderStatus.UNCERTAIN, + updated: undefined, + created: undefined, + action: { system: 'Scrypt', command: 'sell' }, + }); + expect(order.unresolvableTooLong()).toBe(false); + }); + }); + + describe('getAbandonableAt', () => { + function quarantined(minutes?: number, command = 'sell', system = 'Scrypt'): LiquidityManagementOrder { + return Object.assign(new LiquidityManagementOrder(), { + status: LiquidityManagementOrderStatus.UNCERTAIN, + updated: minutes == null ? undefined : minutesAgo(minutes), + action: { system, command }, + }); + } + + /** Milliseconds from now until the order may be given up; negative once it already may be. */ + function headroom(order: LiquidityManagementOrder): number { + const at = order.getAbandonableAt(); + if (!at) throw new Error('expected a deadline'); + + return at.getTime() - Date.now(); + } + + it('is null without a timestamp — no deadline to respect constrains nobody', () => { + expect(quarantined(undefined).getAbandonableAt()).toBeNull(); + }); + + it('falls back to created when updated is missing and returns a Date', () => { + const order = Object.assign(new LiquidityManagementOrder(), { + status: LiquidityManagementOrderStatus.UNCERTAIN, + updated: undefined, + created: minutesAgo(6), + action: { system: 'Scrypt', command: 'sell' }, + }); + expect(order.getAbandonableAt()).toBeInstanceOf(Date); + }); + + it('is null when both updated and created are missing', () => { + const order = Object.assign(new LiquidityManagementOrder(), { + status: LiquidityManagementOrderStatus.UNCERTAIN, + updated: undefined, + created: undefined, + action: { system: 'Scrypt', command: 'sell' }, + }); + expect(order.getAbandonableAt()).toBeNull(); + }); + + it('falls the trade bound after the moment quarantine began', () => { + // 2 of the 5 minutes spent, so 3 left; tolerance covers the clock moving during the test + expect(headroom(quarantined(2))).toBeGreaterThan(2.9 * 60_000); + expect(headroom(quarantined(2))).toBeLessThanOrEqual(3 * 60_000); + }); + + it('lies in the past once the bound has passed, and says by how much', () => { + // The sign is the point: a duration clamped at zero cannot say whether a wait started before the + // deadline, which is exactly what a throttle has to know after it. + expect(headroom(quarantined(6))).toBeLessThan(0); + expect(headroom(quarantined(6))).toBeGreaterThan(-1.1 * 60_000); + expect(headroom(quarantined(60 * 24))).toBeLessThan(-23 * 60 * 60_000); + }); + + it('falls the long bound for a transfer nobody can ask about', () => { + // an hour into a twelve-hour bound leaves eleven + expect(headroom(quarantined(60, 'withdraw', 'Kraken'))).toBeGreaterThan(10.9 * 60 * 60_000); + expect(headroom(quarantined(60, 'withdraw', 'Kraken'))).toBeLessThanOrEqual(11 * 60 * 60_000); + }); + + it('turns over at the bound itself, not a tick later', () => { + // The one instant a live clock cannot be asked about: elapsed exactly equal to the bound. Anything built + // from a real `Date.now()` is already a fraction past it, where `>` and `>=` agree — so the clock is held + // still. With `>` the pass would run (the deadline has arrived) and abandon nothing (not yet past it), + // re-stamp its cooldown, and the order would wait out another interval: five minutes became six. + jest.useFakeTimers(); + try { + const order = quarantined(5); + expect(headroom(order)).toBe(0); + expect(order.unresolvableTooLong()).toBe(true); + + // and one millisecond short of it, both still say no + const justInside = Object.assign(new LiquidityManagementOrder(), { + status: LiquidityManagementOrderStatus.UNCERTAIN, + updated: new Date(Date.now() - (5 * 60_000 - 1)), + action: { system: 'Scrypt', command: 'sell' }, + }); + expect(headroom(justInside)).toBe(1); + expect(justInside.unresolvableTooLong()).toBe(false); + } finally { + jest.useRealTimers(); + } + }); + + it('has passed exactly when the order has become abandonable', () => { + // The two halves of one clock, and they must agree in both directions. The deadline having arrived is what + // lets the reconciliation pass run at all, and being past the bound is what lets it give the order up: a + // state where one holds and the other does not is a pass that runs, re-stamps its cooldown and abandons + // nothing — which is how a five-minute bound came to give up at six. + for (const minutes of [1, 4, 5, 6, 30, 11 * 60, 12 * 60, 13 * 60, 60 * 24]) + for (const command of ['sell', 'buy', 'withdraw']) { + const order = quarantined(minutes, command); + expect(headroom(order) <= 0).toBe(order.unresolvableTooLong()); + } + }); + + it('applies the same bound to the same action as unresolvableTooLong', () => { + // Both read one shared source, so two actions must disagree here exactly where they disagree there: at + // 30 minutes both Scrypt bounds are long spent, while a transfer at a venue that cannot be asked is not. + expect(headroom(quarantined(30, 'sell'))).toBeLessThan(0); + expect(quarantined(30, 'sell').unresolvableTooLong()).toBe(true); + expect(headroom(quarantined(30, 'withdraw'))).toBeLessThan(0); + expect(quarantined(30, 'withdraw').unresolvableTooLong()).toBe(true); + expect(headroom(quarantined(30, 'withdraw', 'Kraken'))).toBeGreaterThan(0); + expect(quarantined(30, 'withdraw', 'Kraken').unresolvableTooLong()).toBe(false); + }); + }); + describe('resolveAsSent / resolveAsNotSent / requestNotSentRelease', () => { it('accepts a release without acting on it: the order keeps blocking', () => { const order = Object.assign(new LiquidityManagementOrder(), { diff --git a/src/subdomains/core/liquidity-management/entities/liquidity-management-order.entity.ts b/src/subdomains/core/liquidity-management/entities/liquidity-management-order.entity.ts index 2c16e613dc..aa8765faf2 100644 --- a/src/subdomains/core/liquidity-management/entities/liquidity-management-order.entity.ts +++ b/src/subdomains/core/liquidity-management/entities/liquidity-management-order.entity.ts @@ -4,7 +4,7 @@ import { IEntity } from 'src/shared/models/entity'; import { Util } from 'src/shared/utils/util'; import { Price, PriceStep } from 'src/subdomains/supporting/pricing/domain/entities/price'; import { Column, Entity, Index, JoinTable, ManyToOne } from 'typeorm'; -import { LiquidityManagementOrderStatus } from '../enums'; +import { LiquidityManagementOrderStatus, LiquidityManagementSystem } from '../enums'; import { OrderFailedException } from '../exceptions/order-failed.exception'; import { OrderNotProcessableException } from '../exceptions/order-not-processable.exception'; import { OrderOutcomeUnknownException } from '../exceptions/order-outcome-unknown.exception'; @@ -19,6 +19,131 @@ import { LiquidityManagementPipeline } from './liquidity-management-pipeline.ent */ const RELEASE_WITHOUT_VENUE_MINUTES = 60; +/** + * How long a quarantined order may go unaccounted for before cleaning it up is worth attempting. + * + * Reaching this bound does not abandon anything by itself — it starts a cancellation. Whether the order is + * then given up depends on the venue settling every reference it claimed, so an order whose integration + * cannot cancel, or whose venue will not answer, stays quarantined past this bound. + * + * The quarantine was built to wait for an operator, on the reasoning that absence at the venue is not proof + * of non-execution. That reasoning is sound but incomplete: it assumed the wait ends. Where nobody checks a + * venue by hand, it does not — the order stays UNCERTAIN forever and takes its rule with it, so the venue + * stops being served at all. A liquidity rule that never runs again is the larger failure, and it is certain, + * while the double execution being guarded against is merely possible. + * + * What carries abandoning is not a conclusion about the order but two things outside it: the venue has + * confirmed that none of its references can execute any more, and a rule replans from the venue's balance + * rather than from the abandoned order. An order that did execute has already moved that balance, so the + * replan sizes itself against what is actually left. That balance is pushed rather than polled, so a fill + * that has only just landed may briefly not be in it — see the adapter's cancellation for that window. + * + * That argument is only as good as the balance behind it, which is why abandoning is confined to orders an + * integration can actually ask the venue about — in practice an exchange, read live at plan time. It is + * deliberately NOT extended to orders no integration can look up: a chain balance omits transactions that + * are sent but unconfirmed, and a bank balance is carried over from the last imported batch, so for those + * the replan could well be sizing itself against a balance the execution has not reached yet. + * + * Within an askable venue, though, an outcome nobody observed at least gets an automatic attempt at cleaning + * itself up, rather than only ever an operator who never comes. Leaving those to a person sounds like the + * careful choice, but where nobody performs the manual release it is not caution, it is a rule that never + * runs again. + * + * "Gets an attempt" is the whole claim, and it is much weaker than "is bounded": the attempt is a cancellation + * (or, for a Scrypt withdrawal that has no cancel, a venue history reply that does not name it), and a + * venue that will not confirm still holds its order here indefinitely. What these bounds end is the assumption + * that somebody will eventually look; the manual release stays a shortcut, not the only way out for venues + * that can answer. + * + * Each bound has to outlast the window in which its order could still be in flight, and that window differs + * by an order of magnitude between kinds of request, so a single value would be either useless or unsafe. + * The two answered bounds are anchored on what completed Scrypt orders actually took over the 30 days to + * 2026-07-29, measured in prod: + * + * trades (n=55): median 9.6s p95 19.8s max 57.1s + * withdrawals (n=49): median 7.7min p95 82min max 5.6h + * + * Read those as a floor, not a ceiling: they describe orders that finished, so one that never becomes + * observable at all is by construction absent from them. Which is why nothing is concluded from the clock + * alone — it only decides when cleaning up is worth attempting. What makes giving up safe is the venue + * confirming that none of the order's references can still execute. + * + * Balances refresh every minute and the pipeline runs every 10 seconds, so no bound is limited by how + * quickly an abandonment can be noticed — only by how long the request itself may still be alive. + */ +const ABANDON_UNCERTAIN_MINUTES = { + /** + * Settled inside the venue, no network leg. Five minutes is roughly five times the slowest such order + * observed, which leaves room for a market phase that keeps one open longer than anything on record. + */ + TRADE: 5, + /** + * A withdrawal at a venue that answers about its own transaction history — today only Scrypt. + * + * Five minutes, and deliberately not derived from how long withdrawals take: measured over 60 days they + * run to 336 minutes, and 39% of them past ten. That tail does not reach this bound, because the bound + * only applies while the venue does NOT name the reference. One it does name is SENT and runs to + * completion on its own clock, but not unbounded — without a transaction hash, + * `SCRYPT_WITHDRAWAL_STUCK_AFTER_MINUTES` (24 hours, in the adapter) fails it so the rule can replan. + * + * What the short value buys is the ceiling the orderer set: five minutes here plus the five a withdrawal + * may stay unobservable (see SCRYPT_UNOBSERVABLE_QUARANTINE_MINUTES) is the ten-minute maximum a + * quarantined order may take to resolve itself. Ten nominal, not to the second: the pass runs on a + * ten-second cron with a few seconds of jitter, and a deadline missed just after a tick waits out the + * one-minute cooldown floor. Read it as ten to eleven minutes plus the venue round-trip. What it costs is + * the case where the venue accepted a withdrawal and publishes it late: it is then given up and reissued. + * That is an internal rebooking — every Scrypt withdrawal address is DFX-owned — and accepted on exactly + * that ground; see `confirmWithdrawalAbsent` for the same trade-off argued at the check itself. + */ + VENUE_WITHDRAWAL: 5, + /** + * Everything else: transfers, withdrawals, bridges, mints. Twice the slowest withdrawal observed, because + * the tail here is genuinely long — a bound near the median would reach orders that are simply still + * running, and reissuing those is what actually moves funds twice. + * + * Reaching this bound only starts the automatic exit attempt — it abandons nothing by itself. For trades + * that attempt is a cancellation whose venue confirmation lets the order go; for a Scrypt withdrawal, + * which has no cancel operation at the venue, it is a confirmed absence from the venue's full transaction + * history. Other transfer kinds and venues that cannot answer that question keep waiting past this bound + * (on the venue, or until an operator releases them as a shortcut). This value is the age at which trying + * is worth it, not a guarantee that giving up is safe. + */ + TRANSFER: 12 * 60, +}; + +/** + * Actions that settle inside a venue rather than moving funds across one, as `system/command` pairs. + * + * Keyed on both halves, because the command name alone does not say what an action does: `sell` on an + * exchange is matched off against a book in seconds, while `sell` on the DEX adapter is an on-chain swap + * with a confirmation time. Matching the name alone would hand the short bound to exactly the actions that + * least deserve it. + * + * An allowlist, not a denylist: anything unrecognised — a new adapter, a renamed command — gets the long + * bound. Being slow to abandon costs a rule some minutes; being fast to abandon a transfer that is still in + * flight is what duplicates it. + * + * The system half comes from the enum so a rename cannot silently detach the list from reality. The command + * half stays literal: those enums live in the adapters, which import this entity, and importing them back + * would close a cycle. + */ +const VENUE_INTERNAL_ACTIONS = [ + `${LiquidityManagementSystem.SCRYPT}/buy`.toLowerCase(), + `${LiquidityManagementSystem.SCRYPT}/sell`.toLowerCase(), +]; + +/** + * Withdrawals at a venue that can be asked about its own transaction history, as `system/command` pairs. + * + * Separate from the general transfer bound on purpose. Lowering TRANSFER itself would also shorten the + * bound for bridges, mints and other systems' transfers — and none of those adapters implements + * `cancelOutstanding`, so no exit would be attempted there anyway; the only effect would be asking their + * venues more often for nothing. + * + * Same allowlist discipline as {@link VENUE_INTERNAL_ACTIONS}: anything unrecognised keeps the long bound. + */ +const VENUE_SELF_RESOLVING_TRANSFERS = [`${LiquidityManagementSystem.SCRYPT}/withdraw`.toLowerCase()]; + @Entity() export class LiquidityManagementOrder extends IEntity { @Column({ length: 256, nullable: false }) @@ -242,6 +367,98 @@ export class LiquidityManagementOrder extends IEntity { return this.notSentRecheckDue != null && Util.minutesDiff(this.notSentRecheckDue) > RELEASE_WITHOUT_VENUE_MINUTES; } + /** + * Whether an unresolved order is old enough that cleaning it up is worth attempting. + * + * Gates both inconclusive outcomes — a venue that answers and has no record, and one that could not be + * asked — because neither improves with waiting. Not a safety judgement on its own: what follows is a + * cancellation, and only the venue confirming that nothing can execute makes giving up safe. See + * {@link ABANDON_UNCERTAIN_MINUTES}. + */ + unresolvableTooLong(): boolean { + // Measured from `updated`, not `created`: an order can run normally for a long time and only become + // UNCERTAIN late, when a completion check amends or restarts it and that write goes unconfirmed. Its + // `created` is then already old, so a bound read from it would expire on the very first pass while the + // fresh replacement request is seconds old and plausibly still arriving — the exact double-send this + // guards against. `updated` moves with the transition into quarantine, so the clock starts there. + // + // When `updated` is missing at runtime (entity `copy()` clears it; some raw loads omit the column) fall + // back to `created` deliberately — not a silent default. `created` is always older than or equal to + // `updated`, so the bound expires earlier rather than later. That is safe here: automatic abandon still + // requires venue confirmation (cancel settled / confirmed absence); this clock only decides *when* a + // cleanup attempt is worth making, not whether giving up is safe. Preferring `updated` when present + // remains correct for the long-running-then-quarantined case above. + // + // Without either timestamp there is no clock to run out. Guarded explicitly because Util.minutesDiff + // treats a missing date as the epoch and would report tens of millions of minutes — abandoning instantly + // every order whose dates were not loaded. Absent evidence this must hold the order, never drop it. + const since = this.updated ?? this.created; + if (!since) return false; + + // `>=`, so that reaching the bound is enough. With `>`, the instant the bound is exactly met left the two + // halves of this clock disagreeing: {@link getAbandonableAt} already reported nothing left — which is + // what lets the reconciliation pass run at that moment — while this said not yet. The pass then re-stamped + // its cooldown and the order waited out another full interval, so a five-minute bound gave up at six. + return Util.minutesDiff(since) >= this.getAbandonBoundMinutes(); + } + + /** + * The moment this order becomes abandonable — null while it never does. + * + * The same bound and the same clock as {@link unresolvableTooLong}, stated as an absolute instant so that a + * caller deciding *when to look at this order again* can keep its own wait on the near side of it. A pass + * throttled past this point would push the abandonment beyond the ceiling that bound exists to impose, and + * nothing in the throttle itself would reveal that it had. + * + * Absolute rather than "time remaining" on purpose. A remaining duration has to be clamped at zero, and that + * clamp erases the one fact a throttle needs after the deadline: whether the wait it is about to impose + * started before it. Measured against a fixed instant the question does not arise — and the caller can ask + * about any moment, not only about now. + * + * Null without `updated` and without `created`, mirroring that method's refusal to run a clock it does not + * have: no deadline to respect means no constraint to impose on a caller's wait. + */ + getAbandonableAt(): Date | null { + // Same clock as {@link unresolvableTooLong}: prefer `updated`, fall back to `created` when `updated` was + // not loaded (see that method for why the fallback is earlier-not-later and still safe). Both missing → + // no deadline, mirroring the refusal to run a clock we do not have. + const since = this.updated ?? this.created; + if (!since) return null; + + return new Date(since.getTime() + this.getAbandonBoundMinutes() * 60_000); + } + + /** + * The quarantine bound that applies to this order's action, in minutes. Deliberately the single place that + * reads {@link ABANDON_UNCERTAIN_MINUTES}: the deadline and the remaining time to it must never be able to + * disagree about which bound they are talking about. + */ + private getAbandonBoundMinutes(): number { + // Unknown, unloaded or unlisted action falls to the long bound, for the same reason as a missing date. + const action = `${this.action?.system}/${this.action?.command}`.toLowerCase(); + + if (VENUE_INTERNAL_ACTIONS.includes(action)) return ABANDON_UNCERTAIN_MINUTES.TRADE; + if (VENUE_SELF_RESOLVING_TRANSFERS.includes(action)) return ABANDON_UNCERTAIN_MINUTES.VENUE_WITHDRAWAL; + + return ABANDON_UNCERTAIN_MINUTES.TRANSFER; + } + + /** + * End a quarantined order that has nothing left outstanding at the venue, and let the rule move on. + * + * FAILED rather than a verified non-execution: nothing here establishes that the request never took + * effect — a reference may well have executed — and the reason says so, so the record does not claim more + * than was actually observed. Named for the state it ends rather than for what ended it, because several + * routes arrive here. + */ + abandonUncertain(reason: string): this { + this.status = LiquidityManagementOrderStatus.FAILED; + this.errorMessage = reason; + this.notSentRecheckDue = null; + + return this; + } + /** * Accept somebody's judgement that this order never left — without acting on it yet. * diff --git a/src/subdomains/core/liquidity-management/enums/index.ts b/src/subdomains/core/liquidity-management/enums/index.ts index edddb69185..6595e46e5d 100644 --- a/src/subdomains/core/liquidity-management/enums/index.ts +++ b/src/subdomains/core/liquidity-management/enums/index.ts @@ -42,6 +42,17 @@ export enum LiquidityManagementOrderStatus { // Quarantine for an order whose request left our side without an observed outcome. Terminal for the // pipeline (it never resumes on its own) but not for the order: `resolveUncertainOrders` asks the venue // what happened and moves it on to IN_PROGRESS or FAILED. See OrderOutcomeUnknownException. + // + // Where an integration can ask the venue, an automatic cleanup is at least attempted: once the order has + // outlived the window in which its request could still be in flight (ABANDON_UNCERTAIN_MINUTES, which + // differs for venue-internal trades and transfers), the adapter is asked to cancel every reference it + // claimed — if it supports cancelling that kind of request at all. + // + // Giving up is never concluded from the clock alone: past the bound the venue is asked to cancel every + // reference the order claimed — sent or merely reserved — and only its answer that none can still execute + // permits FAILED. + // A venue that will not settle them, and an adapter that cannot cancel at all, keep waiting — for + // `resolveUncertainOrderManually`. UNCERTAIN = 'Uncertain', } @@ -51,14 +62,21 @@ export enum UncertainOrderResolution { SENT = 'Sent', /** The venue demonstrably does not know the order — nothing was executed, the rule may plan anew. */ NOT_SENT = 'NotSent', - /** The venue answered, and the answer settles nothing. Stay in quarantine and look again later. */ + /** + * The venue answered, and the answer settles nothing. Stay in quarantine and look again later — until the + * order outlives the abandon bound for its kind of request, at which point the caller tries to cancel + * everything it sent. Only that confirmation releases it; the bound alone never does. + */ UNRESOLVED = 'Unresolved', /** - * The venue could not be asked at all. + * The venue could not be asked, or could not be asked completely. * * Deliberately not the same as UNRESOLVED: that one is an answer, this one is the absence of one, and a * caller that retires an order's outstanding work on the strength of a completed lookup must not retire it * on a failed one. + * + * "Not completely" covers an order with no reference to ask about at all: there is nothing to look up, so + * nothing was learned. */ UNAVAILABLE = 'Unavailable', } diff --git a/src/subdomains/core/liquidity-management/factories/liquidity-action-integration.factory.ts b/src/subdomains/core/liquidity-management/factories/liquidity-action-integration.factory.ts index cd7b6587a8..08ea6beb21 100644 --- a/src/subdomains/core/liquidity-management/factories/liquidity-action-integration.factory.ts +++ b/src/subdomains/core/liquidity-management/factories/liquidity-action-integration.factory.ts @@ -71,4 +71,17 @@ export class LiquidityActionIntegrationFactory { return null; } + + /** + * Resolve the adapter that can talk to the venue for a quarantined order. + * + * Unlike {@link getIntegration}, this ignores `supportedCommands`: reconciling an already-quarantined + * order cares who can ask the venue, not whether the command is still registered as an executable action. + * A command rename or removal must not leave UNCERTAIN rows stranded for an operator forever — the + * adapter for the system still knows how to look up and settle references. {@link getIntegration} stays + * the gate for starting new work; only a registered command may execute. + */ + getReconciliationIntegration(action: LiquidityManagementAction): LiquidityActionIntegration { + return this.adapters.get(action.system) ?? null; + } } diff --git a/src/subdomains/core/liquidity-management/interfaces/index.ts b/src/subdomains/core/liquidity-management/interfaces/index.ts index fed3e41c52..f394437e55 100644 --- a/src/subdomains/core/liquidity-management/interfaces/index.ts +++ b/src/subdomains/core/liquidity-management/interfaces/index.ts @@ -33,6 +33,36 @@ export interface LiquidityActionIntegration { * quarantine for a human to resolve. */ resolveUncertainOrder?(order: LiquidityManagementOrder): Promise; + + /** + * Make sure nothing this order has claimed — sent or merely reserved — can still execute, so it may be + * given up safely. + * + * The one thing that makes abandoning a quarantined order dangerous is a request still live at the venue: + * give the rule its funds back and a late fill spends them twice. Rather than estimating when that can no + * longer happen, this removes the possibility — cancelling is the opposite of re-sending, so it is the one + * write that is always safe against an outcome nobody could observe. Where the venue has no cancel for the + * request kind (a Scrypt withdrawal), the substitute is weaker and knowingly so: the venue answered and did + * not name the reference, with no completeness check on that answer. + * + * Returns a non-empty reason string only when the venue has answered that nothing under this order is left + * to execute (or, for a withdrawal, that a successful history reply does not name it and it did not surface + * in the live cache meanwhile). The caller writes that + * string into the order and the log as-is — each integration supplies its own wording so the pipeline never + * invents a reason the venue never gave. `null` means no automatic exit; the order stays quarantined. + * Read "answered" precisely: a cancellation it accepts, an order it reports terminal, or a successful + * history reply that omits the withdrawal reference settles the question. An unconfirmed cancel must return + * null: it may well have taken effect, but "may well" is what quarantine already means. A truncated history + * is deliberately NOT rejected for withdrawals — that trade-off, and why the alternative was worse, is + * argued where the check lives. + * Integrations that cannot cancel omit this and simply have no automatic exit from quarantine. For Scrypt, + * reconciliation reaches the adapter by system (not by registered command), so every command — including + * one no longer in `supportedCommands` — gets either a venue-confirmed reason string or `null`. Known + * trade/withdraw commands cancel or confirm absence as before; an unsupported command asks `getOrderStatus` + * per reference and cancels any non-terminal one under the symbol that reply itself carries — so "no + * derivable symbol" is not a reason to wait. Neither path waits on an operator as its way out. + */ + cancelOutstanding?(order: LiquidityManagementOrder): Promise; } export interface LiquidityState { diff --git a/src/subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.spec.ts b/src/subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.spec.ts index 3e835685a0..571dc66537 100644 --- a/src/subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.spec.ts +++ b/src/subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.spec.ts @@ -1,5 +1,6 @@ import { createMock } from '@golevelup/ts-jest'; import { NotificationService } from 'src/subdomains/supporting/notification/services/notification.service'; +import { FindOperator } from 'typeorm'; import { LiquidityManagementOrder } from '../entities/liquidity-management-order.entity'; import { LiquidityManagementPipeline } from '../entities/liquidity-management-pipeline.entity'; import { LiquidityManagementRule } from '../entities/liquidity-management-rule.entity'; @@ -12,6 +13,7 @@ import { } from '../enums'; import { OrderOutcomeUnknownException } from '../exceptions/order-outcome-unknown.exception'; import { LiquidityActionIntegrationFactory } from '../factories/liquidity-action-integration.factory'; +import { LiquidityActionIntegration } from '../interfaces'; import { LiquidityManagementOrderRepository } from '../repositories/liquidity-management-order.repository'; import { LiquidityManagementPipelineRepository } from '../repositories/liquidity-management-pipeline.repository'; import { LiquidityManagementRuleRepository } from '../repositories/liquidity-management-rule.repository'; @@ -169,6 +171,7 @@ describe('LiquidityManagementPipelineService', () => { correlationId: 'dfx-lm-9', errorMessage: 'Scrypt did not answer', created: ORDER_CREATED, + updated: new Date(Date.now() - 60_000), action: { id: 233, system: 'Scrypt', command: 'sell' }, ...overrides, }); @@ -184,14 +187,30 @@ describe('LiquidityManagementPipelineService', () => { }); } - function stubIntegration(resolution: UncertainOrderResolution): void { - jest.spyOn(actionIntegrationFactory, 'getIntegration').mockReturnValue({ + /** `cancelSettles` is what the venue says when asked to make sure nothing can execute any more. + * true → reason string (exit), false → null (no exit). The pipeline now receives the reason from the + * integration rather than inventing one. */ + function stubIntegration(resolution: UncertainOrderResolution, cancelSettles = true): LiquidityActionIntegration { + // resolveUncertainOrders looks up the venue via getReconciliationIntegration (system-only). The SENT + // path additionally consults getIntegration: only a registered command may return to IN_PROGRESS. + // Stub both with the same adapter so existing tests model the normal registered-command case they + // always intended (command: 'sell' / supportedCommands: ['sell']). + const integration = { supportedCommands: ['sell'], executeOrder: jest.fn(), checkCompletion: jest.fn(), validateParams: jest.fn(), resolveUncertainOrder: jest.fn().mockResolvedValue(resolution), - }); + cancelOutstanding: jest + .fn() + .mockResolvedValue( + cancelSettles ? 'the venue answered for every reference that nothing is left to execute' : null, + ), + }; + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue(integration); + jest.spyOn(actionIntegrationFactory, 'getIntegration').mockReturnValue(integration); + + return integration; } it('only ever asks about quarantined orders', async () => { @@ -202,6 +221,8 @@ describe('LiquidityManagementPipelineService', () => { expect(findBy).toHaveBeenCalledWith({ status: LiquidityManagementOrderStatus.UNCERTAIN }); }); + // SENT → IN_PROGRESS covers FIX 1 normal case (command still registered via stubIntegration's + // getIntegration mock). The unregistered-command SENT case is asserted separately below. it.each([ [UncertainOrderResolution.SENT, LiquidityManagementOrderStatus.IN_PROGRESS], [UncertainOrderResolution.NOT_SENT, LiquidityManagementOrderStatus.FAILED], @@ -218,6 +239,258 @@ describe('LiquidityManagementPipelineService', () => { expect(order.status).toBe(expectedStatus); }); + /** Quarantined `minutes` ago — the clock runs from `updated`, not from creation. */ + function agedOrder(minutes: number, command = 'sell', system = 'Scrypt'): LiquidityManagementOrder { + return uncertainOrder({ + created: new Date(Date.now() - 30 * 24 * 60 * 60 * 1000), + updated: new Date(Date.now() - minutes * 60 * 1000), + action: { id: 233, system, command } as LiquidityManagementOrder['action'], + }); + } + + function expectResolution( + order: LiquidityManagementOrder, + resolution: UncertainOrderResolution, + ): LiquidityActionIntegration { + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + jest.spyOn(orderRepo, 'update').mockResolvedValue({ affected: 1, raw: [], generatedMaps: [] }); + return stubIntegration(resolution); + } + + // both allowlist entries, so dropping or mistyping either one is caught + it.each(['sell', 'buy'])( + 'abandons a %s past its bound once the venue settles every reference, so its rule is not blocked forever', + async (command) => { + // the failure this prevents: nobody releases the order by hand, so it stays UNCERTAIN indefinitely + // and the rule behind it never plans again — the venue silently stops being served + const order = agedOrder(30, command); + expectResolution(order, UncertainOrderResolution.UNRESOLVED); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.FAILED); + // the record must not claim an observation nobody made + expect(order.errorMessage).toContain('answered for every reference'); + }, + ); + + it('acts on a release immediately when the order carries no reference to look up', async () => { + // an unaskable order reports UNAVAILABLE, so without this the release would sit out the full + // unreachable-venue wait for an answer that can never arrive — somebody already checked by hand + const order = uncertainOrder({ correlationId: undefined, notSentRecheckDue: RELEASED_AT }); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + jest.spyOn(orderRepo, 'update').mockResolvedValue({ affected: 1, raw: [], generatedMaps: [] }); + stubIntegration(UncertainOrderResolution.UNAVAILABLE); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.FAILED); + expect(order.errorMessage).toContain('no reference exists to look it up'); + }); + + it('lets a human release win over the clock when both would apply', async () => { + // Two exits apply here and they record different verdicts: the release says the venue confirmed the + // request never arrived, the abandon says only that nothing is left to execute. The operator's own + // reason survives either way — both prefix the existing message rather than replacing it — but which + // verdict is added to it matters, and only the release rests on somebody having actually checked. The + // branch order decides that, so it is asserted here: reordering the chain later must not quietly file + // an audited case under the weaker of the two. + const order = agedOrder(30); + order.notSentRecheckDue = RELEASED_AT; + order.errorMessage = 'Scrypt did not answer (released by account 42: venue checked — ticket OPS-42)'; + expectResolution(order, UncertainOrderResolution.UNRESOLVED); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.FAILED); + expect(order.errorMessage).toContain('the venue has no record of it either'); + expect(order.errorMessage).not.toContain('abandoned'); + }); + + it('does not abandon while the venue will not settle its references', async () => { + // the only thing that makes giving up dangerous is a request that can still execute. If the venue + // will not confirm that none can — unreachable, or an order it reports in another state — then the + // order keeps waiting. Nothing is concluded from that silence, which is the point. + const order = agedOrder(30); + expectResolution(order, UncertainOrderResolution.UNRESOLVED); + stubIntegration(UncertainOrderResolution.UNRESOLVED, false); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + }); + + it('abandons an order whose references the venue settled, whatever the lookup said', async () => { + // the cancel is what settles it, so an inconclusive lookup is no obstacle: once nothing can execute, + // giving up is a fact rather than an estimate + const order = agedOrder(30); + expectResolution(order, UncertainOrderResolution.UNAVAILABLE); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.FAILED); + }); + + it('keeps a trade quarantined inside its bound — the slowest observed trade took under a minute', async () => { + const order = agedOrder(1); + expectResolution(order, UncertainOrderResolution.UNRESOLVED); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + }); + + it('holds a transfer nobody can ask about far longer than a Scrypt one', async () => { + // Scrypt withdrawals now share the short bound: the venue answers about its own history, and the + // ten-minute ceiling outweighs the long tail, since a reissued payout only moves funds between our own + // accounts. A bridge or mint has neither property, so the long bound stays where it is. + const order = agedOrder(30, 'withdraw', 'Kraken'); + expectResolution(order, UncertainOrderResolution.UNRESOLVED); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + }); + + it('abandons a Scrypt withdrawal past its own five-minute bound, not the transfer one', async () => { + // the failure this prevents: a typo or case mismatch in the allowlist that maps Scrypt withdrawals + // to their five-minute bound would leave them on the long transfer bound unnoticed. + const order = agedOrder(6, 'withdraw'); + const integration = expectResolution(order, UncertainOrderResolution.UNRESOLVED); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.FAILED); + expect(integration.cancelOutstanding).toHaveBeenCalledWith(order); + }); + + it('keeps a Scrypt withdrawal quarantined while it is still inside its own five-minute bound', async () => { + // the failure this prevents: cancelOutstanding would be called and the withdrawal abandoned before + // its own five-minute bound has run out. + const order = agedOrder(4, 'withdraw'); + const integration = expectResolution(order, UncertainOrderResolution.UNRESOLVED); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + expect(integration.cancelOutstanding).not.toHaveBeenCalled(); + }); + + it('abandons a transfer once even its long bound has run out and the venue settles it', async () => { + // the bound alone is not enough: this asserts the pipeline's side of the contract, that an aged + // transfer whose integration returns a reason string from cancelOutstanding does get abandoned. + // How the integration settles the question (trade cancel vs. a withdrawal history reply that does not name it) + // is its business — the pipeline only forwards the returned reason. + const order = agedOrder(13 * 60, 'withdraw'); + expectResolution(order, UncertainOrderResolution.UNRESOLVED); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.FAILED); + expect(order.errorMessage).toContain('answered for every reference'); + }); + + it('gives an unrecognised command the long bound, not the short one', async () => { + // allowlist, not denylist: a new adapter must not inherit the trade bound by accident + const order = agedOrder(30, 'some-new-bridge-command'); + expectResolution(order, UncertainOrderResolution.UNRESOLVED); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + }); + + it('never abandons an order no integration can look up, however old', async () => { + // the venue is never asked here, so only the clock would be left — and the safety of abandoning rests + // on replanning against a balance that reflects an execution. A chain balance omits unconfirmed + // transactions and a bank balance comes from the last import, so that does not hold off-exchange. + const order = agedOrder(30 * 24 * 60); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + jest.spyOn(orderRepo, 'update').mockResolvedValue({ affected: 1, raw: [], generatedMaps: [] }); + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue(undefined); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + }); + + it('gives an on-chain swap the long bound even though its command is called "sell"', async () => { + // the command name alone does not say what an action does: DfxDex/sell is an on-chain swap with a + // confirmation time, not a book match settled in seconds + const order = agedOrder(30, 'sell', 'DfxDex'); + expectResolution(order, UncertainOrderResolution.UNRESOLVED); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + }); + + it('narrows an abandon on the absent release as SQL NULL, not as a bare null', async () => { + // the abandon concludes nothing about the send itself, so it must not outrank an operator who checked and is + // still owed one venue answer — hence the narrowing. But "no release pending" is the ordinary case + // here, and a raw null renders as `= NULL`, which matches no row at all: the update would never + // affect anything and would report itself as a lost race. A mocked repo cannot see that, so assert + // on the operator itself. + const order = agedOrder(30); + const update = jest.spyOn(orderRepo, 'update').mockResolvedValue({ affected: 1, raw: [], generatedMaps: [] }); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + stubIntegration(UncertainOrderResolution.UNRESOLVED); + + await service['resolveUncertainOrders'](); + + const criteria = update.mock.calls[0][0] as { notSentRecheckDue: FindOperator }; + expect(criteria.notSentRecheckDue).toBeInstanceOf(FindOperator); + expect(criteria.notSentRecheckDue.type).toBe('isNull'); + expect(order.status).toBe(LiquidityManagementOrderStatus.FAILED); + }); + + it('narrows a not-sent verdict on the absent release as SQL NULL too', async () => { + // the same trap on the other caller: a venue verdict arrives with nobody having released the order, + // so its examined value is null as well. Asserting on status alone would not catch a regression here, + // because resolveAsNotSent already sets FAILED synchronously before the write is attempted. + const order = uncertainOrder(); + const update = jest.spyOn(orderRepo, 'update').mockResolvedValue({ affected: 1, raw: [], generatedMaps: [] }); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + stubIntegration(UncertainOrderResolution.NOT_SENT); + + await service['resolveUncertainOrders'](); + + const criteria = update.mock.calls[0][0] as { notSentRecheckDue: FindOperator }; + expect(criteria.notSentRecheckDue).toBeInstanceOf(FindOperator); + expect(criteria.notSentRecheckDue.type).toBe('isNull'); + }); + + it('runs the abandon clock off `created` once `updated` is missing, because created is always the older (or equal) bound', async () => { + // entity copy() clears `updated`, and some raw loads omit the column — `created` stays the only + // clock left. It is never younger than `updated` would have been, so falling back to it can only make + // the bound expire earlier, never later: the one direction that keeps giving up safe. + const order = uncertainOrder({ updated: undefined }); + expectResolution(order, UncertainOrderResolution.UNRESOLVED); + + await service['resolveUncertainOrders'](); + + // ORDER_CREATED is 29 days old, decisively past the 5-minute trade bound + expect(order.status).toBe(LiquidityManagementOrderStatus.FAILED); + }); + + it('never abandons an order with neither a quarantine timestamp nor a creation date', async () => { + // Util.minutesDiff reads a missing date as the epoch; unguarded that abandons instantly. `created` is + // the last fallback, and losing that too must leave no clock at all — the order can only wait. + // + // Routed through a pending release (rather than a bare uncertainOrder()) so the pass takes the + // releasePending branch: that branch never reads order.created for the cooldown throttle, so this + // reaches unresolvableTooLong()'s own missing-date guard instead of the unrelated crash a bare + // uncertain order with no `created` would hit in the age-based throttle a few lines above it. + const order = uncertainOrder({ updated: undefined, created: undefined, notSentRecheckDue: RELEASED_AT }); + stubIntegration(UncertainOrderResolution.UNAVAILABLE); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + expect(orderRepo.update).not.toHaveBeenCalled(); + }); + it('keeps a released order quarantined until the venue has actually answered', async () => { // the release is a judgement, and while it is unconfirmed the order must not become terminal — // a terminal order lets its rule plan again against funds that may well be committed @@ -266,6 +539,23 @@ describe('LiquidityManagementPipelineService', () => { expect(update.mock.calls[0][0]).toMatchObject({ notSentRecheckDue: RELEASED_AT }); }); + it('does not report a missed quarantine write as a resolution that happened elsewhere', async () => { + // The write cannot tell a race from a narrowing that will never match again, and the second case repeats + // forever while the order stays as it is. Claiming the benign one hid exactly that: a release timestamp + // with microsecond precision is unmatchable by a JS Date, and it held a live order for hours behind the + // reassuring wording. So the line must name both and be a warning, not routine information. + const order = releasePendingOrder(); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + jest.spyOn(orderRepo, 'update').mockResolvedValue({ affected: 0, raw: [], generatedMaps: [] }); + const warn = jest.spyOn(service['logger'], 'warn').mockImplementation(() => undefined); + stubIntegration(UncertainOrderResolution.UNRESOLVED); + + await service['resolveUncertainOrders'](); + + expect(warn).toHaveBeenCalledWith(expect.stringMatching(/was not updated.*resolved elsewhere.*matched no row/s)); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('stuck')); + }); + it('does not release an unreleased order on the same inconclusive answer', async () => { // absence is not proof; without somebody having checked independently there is only one negative const order = uncertainOrder(); @@ -308,7 +598,7 @@ describe('LiquidityManagementPipelineService', () => { const order = releasePendingOrder(); jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); const update = jest.spyOn(orderRepo, 'update').mockResolvedValue({ affected: 1, raw: [], generatedMaps: [] }); - jest.spyOn(actionIntegrationFactory, 'getIntegration').mockReturnValue(null); + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue(null); // and the pass reports the change, so the caller's loop knows something moved await expect(service['resolveUncertainOrders']()).resolves.toBe(true); @@ -322,7 +612,7 @@ describe('LiquidityManagementPipelineService', () => { it('leaves an unreleased order alone when its adapter is gone', async () => { const order = uncertainOrder(); jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); - jest.spyOn(actionIntegrationFactory, 'getIntegration').mockReturnValue(null); + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue(null); await service['resolveUncertainOrders'](); @@ -470,7 +760,7 @@ describe('LiquidityManagementPipelineService', () => { it('keeps the order quarantined when the lookup itself throws', async () => { const order = uncertainOrder(); jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); - jest.spyOn(actionIntegrationFactory, 'getIntegration').mockReturnValue({ + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue({ supportedCommands: ['sell'], executeOrder: jest.fn(), checkCompletion: jest.fn(), @@ -483,6 +773,150 @@ describe('LiquidityManagementPipelineService', () => { expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); }); + it('reaches the adapter for a Scrypt order whose command is no longer registered', async () => { + // getIntegration would return null for an unregistered command; reconciliation resolves by system + const resolveUncertainOrder = jest.fn().mockResolvedValue(UncertainOrderResolution.UNRESOLVED); + const cancelOutstanding = jest.fn().mockResolvedValue(null); + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue({ + supportedCommands: ['sell', 'buy', 'withdraw'], // deliberately omits the order's command + executeOrder: jest.fn(), + checkCompletion: jest.fn(), + validateParams: jest.fn(), + resolveUncertainOrder, + cancelOutstanding, + }); + // getIntegration would skip — prove reconciliation does not use it for this path + jest.spyOn(actionIntegrationFactory, 'getIntegration').mockReturnValue(null); + const order = agedOrder(30, 'sell-if-deficit', 'Scrypt'); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + jest.spyOn(orderRepo, 'update').mockResolvedValue({ affected: 1, raw: [], generatedMaps: [] }); + + await service['resolveUncertainOrders'](); + + expect(resolveUncertainOrder).toHaveBeenCalledWith(order); + }); + + it('keeps a venue-confirmed SENT order quarantined when its command is no longer registered', async () => { + // Venue knows the reference, but no registered command can checkCompletion. Returning to IN_PROGRESS + // would trap the order (see FIX 1); stay UNCERTAIN so the automatic abandon path remains available. + const resolveUncertainOrder = jest.fn().mockResolvedValue(UncertainOrderResolution.SENT); + const cancelOutstanding = jest.fn().mockResolvedValue(null); + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue({ + supportedCommands: ['sell', 'buy', 'withdraw'], // deliberately omits the order's command + executeOrder: jest.fn(), + checkCompletion: jest.fn(), + validateParams: jest.fn(), + resolveUncertainOrder, + cancelOutstanding, + }); + jest.spyOn(actionIntegrationFactory, 'getIntegration').mockReturnValue(null); + const order = agedOrder(30, 'sell-if-deficit', 'Scrypt'); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + const warn = jest.spyOn(service['logger'], 'warn').mockImplementation(() => undefined); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + expect(order.status).not.toBe(LiquidityManagementOrderStatus.IN_PROGRESS); + expect(resolveUncertainOrder).toHaveBeenCalledWith(order); + expect(warn).toHaveBeenCalledWith(expect.stringMatching(/stays quarantined.*no registered command/s)); + expect(cancelOutstanding).not.toHaveBeenCalled(); + }); + + it('abandons a venue-confirmed SENT order past its bound when its command is no longer registered', async () => { + const resolveUncertainOrder = jest.fn().mockResolvedValue(UncertainOrderResolution.SENT); + const cancelOutstanding = jest + .fn() + .mockResolvedValue('the venue answered for every reference that nothing is left to execute'); + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue({ + supportedCommands: ['sell', 'buy', 'withdraw'], // deliberately omits the order's command + executeOrder: jest.fn(), + checkCompletion: jest.fn(), + validateParams: jest.fn(), + resolveUncertainOrder, + cancelOutstanding, + }); + jest.spyOn(actionIntegrationFactory, 'getIntegration').mockReturnValue(null); + const order = agedOrder(30, 'sell', 'Scrypt'); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + jest.spyOn(orderRepo, 'update').mockResolvedValue({ affected: 1, raw: [], generatedMaps: [] }); + jest.spyOn(service['logger'], 'warn').mockImplementation(() => undefined); + + await service['resolveUncertainOrders'](); + + expect(cancelOutstanding).toHaveBeenCalledWith(order); + expect(order.status).toBe(LiquidityManagementOrderStatus.FAILED); + expect(order.errorMessage).toContain('the venue answered for every reference that nothing is left to execute'); + }); + + it('keeps a venue-confirmed SENT order past its bound quarantined when cancelOutstanding is unsettled', async () => { + const resolveUncertainOrder = jest.fn().mockResolvedValue(UncertainOrderResolution.SENT); + const cancelOutstanding = jest.fn().mockResolvedValue(null); + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue({ + supportedCommands: ['sell', 'buy', 'withdraw'], // deliberately omits the order's command + executeOrder: jest.fn(), + checkCompletion: jest.fn(), + validateParams: jest.fn(), + resolveUncertainOrder, + cancelOutstanding, + }); + jest.spyOn(actionIntegrationFactory, 'getIntegration').mockReturnValue(null); + const order = agedOrder(30, 'sell', 'Scrypt'); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + jest.spyOn(service['logger'], 'warn').mockImplementation(() => undefined); + + await service['resolveUncertainOrders'](); + + expect(cancelOutstanding).toHaveBeenCalledWith(order); + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + }); + + it('abandons a released venue-confirmed SENT order past its bound via cancelOutstanding, not completeNotSentRelease', async () => { + const resolveUncertainOrder = jest.fn().mockResolvedValue(UncertainOrderResolution.SENT); + const cancelOutstanding = jest + .fn() + .mockResolvedValue('the venue answered for every reference that nothing is left to execute'); + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue({ + supportedCommands: ['sell', 'buy', 'withdraw'], // deliberately omits the order's command + executeOrder: jest.fn(), + checkCompletion: jest.fn(), + validateParams: jest.fn(), + resolveUncertainOrder, + cancelOutstanding, + }); + jest.spyOn(actionIntegrationFactory, 'getIntegration').mockReturnValue(null); + const order = agedOrder(30, 'sell', 'Scrypt'); + order.notSentRecheckDue = RELEASED_AT; + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + jest.spyOn(orderRepo, 'update').mockResolvedValue({ affected: 1, raw: [], generatedMaps: [] }); + jest.spyOn(service['logger'], 'warn').mockImplementation(() => undefined); + + await service['resolveUncertainOrders'](); + + expect(cancelOutstanding).toHaveBeenCalledWith(order); + expect(order.status).toBe(LiquidityManagementOrderStatus.FAILED); + expect(order.errorMessage).toContain('the venue answered for every reference that nothing is left to execute'); + expect(order.errorMessage).not.toContain('the venue confirmed the request never arrived'); + }); + + it('leaves a non-Scrypt system without resolveUncertainOrder alone (no automatic progress)', async () => { + // observable behaviour unchanged vs getIntegration: adapter exists but offers no reconciliation + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue({ + supportedCommands: ['transfer'], + executeOrder: jest.fn(), + checkCompletion: jest.fn(), + validateParams: jest.fn(), + // no resolveUncertainOrder + }); + const order = agedOrder(30 * 24 * 60, 'transfer', 'SomeBank'); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + expect(orderRepo.update).not.toHaveBeenCalled(); + }); + describe('venue-lookup cooldown', () => { // The cooldown is a pure function of Date.now(), so these tests drive the clock instead of waiting. // Scoped here rather than suite-wide: nothing else in this file cares about time. @@ -494,7 +928,7 @@ describe('LiquidityManagementPipelineService', () => { * and only the cooldown decides whether the venue is asked. */ function stubResolver(): jest.Mock { const resolveUncertainOrder = jest.fn().mockResolvedValue(UncertainOrderResolution.UNAVAILABLE); - jest.spyOn(actionIntegrationFactory, 'getIntegration').mockReturnValue({ + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue({ supportedCommands: ['sell'], executeOrder: jest.fn(), checkCompletion: jest.fn(), @@ -615,7 +1049,12 @@ describe('LiquidityManagementPipelineService', () => { // 11 min 20 s is past it. A one-sided assertion would let the rate drift unnoticed: with `ageMs / 5` // the order simply stays in cooldown and a lower-bound-only test keeps passing. const resolveUncertainOrder = stubResolver(); - const order = uncertainOrder({ created: new Date(Date.now() - 100 * 60_000) }); + const order = uncertainOrder({ + created: new Date(Date.now() - 100 * 60_000), + // A venue that cannot be asked keeps the twelve-hour bound, so no deadline tightens the interval + // and the age formula alone is under test here. + action: { id: 233, system: 'Kraken', command: 'withdraw' } as LiquidityManagementOrder['action'], + }); jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); await service['resolveUncertainOrders'](); @@ -629,13 +1068,119 @@ describe('LiquidityManagementPipelineService', () => { expect(resolveUncertainOrder).toHaveBeenCalledTimes(2); }); + it('holds the deadline cap still between lookups instead of letting it shrink', async () => { + // The cap is the time that was left when the last lookup finished. Read fresh on every tick it would + // shrink while the wait grows, the two would meet halfway, and a five-minute bound would be re-asked + // at 2.5 minutes — then 3.75, then 4.4, a geometric series of expensive lookups before a deadline that + // never moved. Asserted just before the bound, where the shrinking variant asks and this one does not. + const order = agedOrder(0); + const resolveUncertainOrder = jest.fn().mockResolvedValue(UncertainOrderResolution.UNAVAILABLE); + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue({ + supportedCommands: ['sell'], + executeOrder: jest.fn(), + checkCompletion: jest.fn(), + validateParams: jest.fn(), + resolveUncertainOrder, + }); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + + await service['resolveUncertainOrders'](); + expect(resolveUncertainOrder).toHaveBeenCalledTimes(1); + + // half the bound: exactly where a cap re-read on this tick would coincide with the elapsed wait + jest.advanceTimersByTime(2.5 * 60_000); + await service['resolveUncertainOrders'](); + expect(resolveUncertainOrder).toHaveBeenCalledTimes(1); + + // one millisecond short of the bound — still nothing to give up, so still nothing to ask + jest.advanceTimersByTime(2.5 * 60_000 - 1); + await service['resolveUncertainOrders'](); + expect(resolveUncertainOrder).toHaveBeenCalledTimes(1); + + // and exactly at the bound it asks, which is what the cap exists to guarantee + jest.advanceTimersByTime(1); + await service['resolveUncertainOrders'](); + expect(resolveUncertainOrder).toHaveBeenCalledTimes(2); + }); + + it('does not let its own floor push the abandonment past the deadline', async () => { + // A lookup that runs shortly before the bound has less than a floor's worth of time left. Raising the cap + // to the floor there schedules the next pass after the deadline — the overshoot the cap exists to prevent, + // caused by the cap. Reached the way it happens in practice: the order enters this pass already close to + // its bound, with no cooldown recorded, so the first lookup lands 30 seconds short of it. + const order = agedOrder(4.5); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + jest.spyOn(orderRepo, 'update').mockResolvedValue({ affected: 1, raw: [], generatedMaps: [] }); + stubIntegration(UncertainOrderResolution.UNRESOLVED); + + // first lookup at 4:30, which stamps the cooldown with only 30 seconds of headroom left + await service['resolveUncertainOrders'](); + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + + // at the bound the order must be gone — a floor measured from 4:30 would hold it until 5:30 + jest.advanceTimersByTime(30_000); + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.FAILED); + }); + + it('never throttles past the bound at which the order becomes abandonable', async () => { + // Where the cooldown and the abandon bound meet. This same pass is what gives an expired order up, so + // a wait longer than what is left of its bound postpones the abandonment — an order weeks old draws + // the full thirty-minute interval, six times a trade's own five-minute bound, and the ceiling this + // branch exists to impose would have been raised with nothing saying so. + const order = agedOrder(0); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + jest.spyOn(orderRepo, 'update').mockResolvedValue({ affected: 1, raw: [], generatedMaps: [] }); + stubIntegration(UncertainOrderResolution.UNRESOLVED); + + // inside the bound: nothing to give up yet + await service['resolveUncertainOrders'](); + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + + // past the bound, still far short of the thirty-minute cap that would otherwise still be running + jest.advanceTimersByTime(6 * 60_000); + await service['resolveUncertainOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.FAILED); + }); + + it("leaves the cap governing when the order's own bound is further off than the cap", async () => { + // The deadline only ever tightens the interval, never loosens it: a transfer has twelve hours, so the + // cap decides exactly as it did before, and a lookup one millisecond early still must not happen. + const order = agedOrder(0, 'withdraw', 'Kraken'); + const resolveUncertainOrder = jest.fn().mockResolvedValue(UncertainOrderResolution.UNAVAILABLE); + jest.spyOn(actionIntegrationFactory, 'getReconciliationIntegration').mockReturnValue({ + supportedCommands: ['withdraw'], + executeOrder: jest.fn(), + checkCompletion: jest.fn(), + validateParams: jest.fn(), + resolveUncertainOrder, + }); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + + await service['resolveUncertainOrders'](); + + jest.advanceTimersByTime(30 * 60_000 - 1); + await service['resolveUncertainOrders'](); + expect(resolveUncertainOrder).toHaveBeenCalledTimes(1); + + jest.advanceTimersByTime(1); + await service['resolveUncertainOrders'](); + expect(resolveUncertainOrder).toHaveBeenCalledTimes(2); + }); + it('caps the cooldown interval at thirty minutes no matter how old the order is', async () => { // Pins the cap to the millisecond. An 8-hour-old order's uncapped wait would be 48 minutes at the // first pass and 51 by the time of the boundary check — either way far past the cap, so a lookup at // exactly 30 minutes can only come from it. Requiring no lookup a millisecond earlier leaves the cap // no other whole-millisecond value to take, and landing on the boundary pins `<` against `<=`. const resolveUncertainOrder = stubResolver(); - const order = uncertainOrder({ created: new Date(Date.now() - 8 * 60 * 60_000) }); + const order = uncertainOrder({ + created: new Date(Date.now() - 8 * 60 * 60_000), + // Long bound on purpose (see above): this pins the thirty-minute cap, not a deadline. + action: { id: 233, system: 'Kraken', command: 'withdraw' } as LiquidityManagementOrder['action'], + }); jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); await service['resolveUncertainOrders'](); @@ -651,6 +1196,29 @@ describe('LiquidityManagementPipelineService', () => { }); }); + describe('checkRunningOrders', () => { + it('quarantines a running order whose command is no longer registered', async () => { + // Without the null guard in checkOrder this would TypeError, land only in logger.error, and leave the + // order stuck in IN_PROGRESS with no automatic or manual exit. OrderOutcomeUnknownException is the + // same path startNewOrders already uses for unknown outcomes — quarantine, not a hang. + const order = Object.assign(new LiquidityManagementOrder(), { + id: 11, + status: LiquidityManagementOrderStatus.IN_PROGRESS, + correlationId: 'dfx-lm-11', + action: { id: 233, system: 'Scrypt', command: 'sell-if-deficit' }, + }); + jest.spyOn(orderRepo, 'findBy').mockResolvedValue([order]); + jest.spyOn(orderRepo, 'save').mockImplementation(async (o: LiquidityManagementOrder) => o); + jest.spyOn(actionIntegrationFactory, 'getIntegration').mockReturnValue(null); + + await service['checkRunningOrders'](); + + expect(order.status).toBe(LiquidityManagementOrderStatus.UNCERTAIN); + expect(order.errorMessage).toMatch(/no registered integration.*Scrypt\/sell-if-deficit/s); + expect(notificationService.sendMail).toHaveBeenCalled(); + }); + }); + describe('resolveUncertainOrderManually', () => { const VERIFIED_DTO = { noExecutionVerified: true, verificationReference: 'venue console, ticket OPS-42' }; diff --git a/src/subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.ts b/src/subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.ts index a25617b6ba..8c74b457fe 100644 --- a/src/subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.ts +++ b/src/subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.ts @@ -7,7 +7,7 @@ import { Util } from 'src/shared/utils/util'; import { MailContext, MailType } from 'src/subdomains/supporting/notification/enums'; import { MailRequest } from 'src/subdomains/supporting/notification/interfaces'; import { NotificationService } from 'src/subdomains/supporting/notification/services/notification.service'; -import { In } from 'typeorm'; +import { In, IsNull } from 'typeorm'; import { ResolveUncertainOrderDto } from '../dto/resolve-uncertain-order.dto'; import { LiquidityManagementOrder } from '../entities/liquidity-management-order.entity'; import { LiquidityManagementPipeline } from '../entities/liquidity-management-pipeline.entity'; @@ -17,6 +17,7 @@ import { OrderNotNecessaryException } from '../exceptions/order-not-necessary.ex import { OrderNotProcessableException } from '../exceptions/order-not-processable.exception'; import { OrderOutcomeUnknownException } from '../exceptions/order-outcome-unknown.exception'; import { LiquidityActionIntegrationFactory } from '../factories/liquidity-action-integration.factory'; +import { LiquidityActionIntegration } from '../interfaces'; import { LiquidityManagementOrderRepository } from '../repositories/liquidity-management-order.repository'; import { LiquidityManagementPipelineRepository } from '../repositories/liquidity-management-pipeline.repository'; import { LiquidityManagementRuleRepository } from '../repositories/liquidity-management-rule.repository'; @@ -321,10 +322,28 @@ export class LiquidityManagementPipelineService { /** * Resolve orders quarantined as UNCERTAIN by asking the venue what actually happened. * - * This only ever observes — it must not re-send anything. An order leaves quarantine when the venue - * either confirms it knows the reference (back to IN_PROGRESS, the normal completion check takes over) or - * demonstrably does not (FAILED, so the rule may plan anew from a fresh balance). Anything inconclusive - * stays put: an order nobody can account for is safer parked than retried. + * This only ever observes or cancels — it must never re-send anything. An order leaves quarantine when the venue + * either confirms it knows the reference *and* a registered command still exists to check its completion + * (back to IN_PROGRESS, the normal completion check takes over), or demonstrably does not (FAILED, so the + * rule may plan anew from a fresh balance). A venue-side SENT for a command that is no longer registered + * deliberately stays quarantined: without a completion check, returning it to IN_PROGRESS would leave it + * with no exit, so the way out remains the existing automatic cancel/abandon path + * (`cancelOutstanding` / `unresolvableTooLong`) — not an operator. Anything inconclusive stays put, and past + * the abandon bound for its kind of request a cancellation is attempted — because a rule parked forever is + * the worse failure. For a cancellable request it is given up as FAILED only once the venue has confirmed + * that nothing under this order can still execute. A Scrypt withdrawal cannot be cancelled at all, so there + * the bar is lower by design: the venue answered and did not name the reference, without any completeness + * check on that answer — a repeated payout goes to a DFX-owned address, whereas insisting on completeness + * would strand the order. Age decides when it is worth trying to clean up; what the venue says decides + * whether giving up is safe. So the bound is not a deadline after which the order is certainly gone. For Scrypt + * (and any system whose adapter implements `resolveUncertainOrder`), an order whose references the venue + * will not yet settle keeps waiting past it — on the venue, not on an operator: reconciliation resolves by + * system, so a renamed or removed command still reaches the adapter and can be observed or cancelled there. + * Returning that order to the normal pipeline, however, still requires `getIntegration` (registered command). + * Systems whose adapter omits `resolveUncertainOrder` have no automatic venue path; without a pending + * release they are skipped every pass until an operator acts (`releasePending`). An operator can still + * release sooner as a shortcut; where the adapter can ask, the mechanism that ends the wait is the venue + * answering (or confirming absence) on a later pass. */ private async resolveUncertainOrders(): Promise { // First: anything this process observed and could not write. Retried before new lookups, because an @@ -348,20 +367,43 @@ export class LiquidityManagementPipelineService { // pending release bypasses the wait entirely: a manual release must complete on the next tick, and // its own venue-wait runs on its own clock. if (!releasePending) { + const lastAttemptEnd = this.uncertainResolveAttempts.get(order.id); + const abandonableAt = order.getAbandonableAt(); const ageMs = Date.now() - order.created.getTime(); + + // How long the last lookup had left before this order became abandonable. Negative once that lookup + // itself ran after the deadline, and Infinity when there is no deadline or no previous lookup to + // measure from. + const deadlineHeadroomMs = + abandonableAt && lastAttemptEnd ? abandonableAt.getTime() - lastAttemptEnd.getTime() : Infinity; + const intervalMs = Math.min( Math.max(ageMs / 10, UNCERTAIN_RESOLVE_MIN_INTERVAL_MS), UNCERTAIN_RESOLVE_MAX_INTERVAL_MS, + // The throttle may never outlast the deadline it has to keep. This same pass is what abandons an + // order whose bound has run out, so a wait reaching past that bound postpones the abandonment beyond + // the ceiling the bound exists to impose — a trade quarantined when it was already eight hours old + // would be given up after thirty minutes instead of five, with nothing here saying so. + // + // Taken as-is while there is headroom, deliberately without the floor: a lookup that ended shortly + // before the deadline has less than a floor's worth of time left, and imposing a full interval on it + // would schedule the next pass after the deadline — the very overshoot this cap prevents, caused by + // the cap. It costs at most one extra lookup, because the pass it permits is the one that gives the + // order up. + // + // Past the deadline the floor governs instead: there is no deadline left to protect, and a + // cancellation the venue will not confirm must not retry on every ten-second tick. + deadlineHeadroomMs > 0 ? deadlineHeadroomMs : UNCERTAIN_RESOLVE_MIN_INTERVAL_MS, ); - const lastAttemptEnd = this.uncertainResolveAttempts.get(order.id); if (lastAttemptEnd && Date.now() - lastAttemptEnd.getTime() < intervalMs) continue; } try { - // Null when the action's system or command is no longer registered at all — an order can outlive the - // adapter that made it, and dereferencing that would throw here on every pass, forever. - const actionIntegration = this.actionIntegrationFactory.getIntegration(order.action); + // Null only when the action's *system* has no adapter at all. Command registration is ignored here: + // a quarantined order can outlive a command rename/removal, and reconciliation still needs whoever + // can ask that venue. Execution of new orders keeps using getIntegration (registered commands only). + const actionIntegration = this.actionIntegrationFactory.getReconciliationIntegration(order.action); if (!actionIntegration?.resolveUncertainOrder) { // The one exception to "a release waits for the venue": there is no lookup for this order at all, @@ -369,6 +411,12 @@ export class LiquidityManagementPipelineService { // operator's judgement is all there is, which is why the assertion behind it is required. if (releasePending && (await this.completeNotSentRelease(order, 'no integration can look it up'))) anyChanged = true; + // Deliberately no automatic abandon here. Without an integration the venue is never asked, so the + // only thing left would be the clock — and the safety of abandoning rests on the rule replanning + // from a balance that reflects an execution which did happen. That holds for an exchange read live + // at plan time; it does not hold for a chain balance that omits unconfirmed transactions, nor for a + // bank balance carried over from the last imported batch. Abandoning on the clock alone would be + // guessing with the one class of order nothing here can observe. continue; } @@ -383,7 +431,30 @@ export class LiquidityManagementPipelineService { } if (resolution === UncertainOrderResolution.SENT) { - if (await this.applyConfirmedObservation(order)) { + // Reconciliation looked up by system alone so an unregistered command can still be *observed* — + // that is intentional and stays that way (see getReconciliationIntegration above). Putting the + // order back into IN_PROGRESS is a different decision: the normal completion path uses the strict + // getIntegration (registered commands only). An order whose command is gone would land in + // IN_PROGRESS with no adapter that can checkCompletion — every checkRunningOrders pass would + // TypeError, never quarantine, and the automatic abandon path would never run. Observing is + // allowed for anyone who can ask the venue; advancing out of quarantine is only allowed for + // whoever can also finish the job. Leave it UNCERTAIN so cancelOutstanding / unresolvableTooLong + // still apply once the bound is reached. + if (!this.actionIntegrationFactory.getIntegration(order.action)) { + this.logger.warn( + `Uncertain liquidity order ${order.id} stays quarantined: venue confirmed it was sent ` + + `(system ${order.action.system}, command ${order.action.command}), but no registered command ` + + `can check its completion — returning it to IN_PROGRESS would leave it with no exit`, + ); + + // A SENT for a command that no longer exists is a real observation ("the reference exists") but + // not one that can return to normal operation. The only remaining exit is the same as for an + // unresolvable case: after the bound elapses, cancel and abandon. resolveUncertainOrder answers + // "does the reference exist", not "is it still open" — so SENT stays SENT permanently, and the + // cleanup path must not depend on a future status change; it has to run independently of whether + // the command is still registered. + if (await this.attemptQuarantineCleanup(order, actionIntegration)) anyChanged = true; + } else if (await this.applyConfirmedObservation(order)) { anyChanged = true; this.logger.info(`Uncertain liquidity order ${order.id} resolved: venue confirmed it was sent`); } @@ -396,14 +467,23 @@ export class LiquidityManagementPipelineService { // checked independently and released the order on that basis. Two negatives, one of them from a // person who looked: that is what this release was waiting for. if (await this.completeNotSentRelease(order, 'the venue has no record of it either')) anyChanged = true; + } else if (releasePending && !order.correlationId) { + // Nothing ever went out under a reference, so no lookup can produce an answer — the same dead end + // as an order with no integration, and the same reason not to make somebody who already checked + // wait out a clock. Without this the release would sit through the full unreachable-venue wait, + // because an unaskable order now reports UNAVAILABLE rather than an answer. + if (await this.completeNotSentRelease(order, 'no reference exists to look it up')) anyChanged = true; } else if (releasePending && order.releaseWaitedOutVenue()) { // Nobody has been able to ask this venue anything for long enough. Waiting more does not make an // answer likelier; it only keeps an order a person has verified by hand out of reach. if (await this.completeNotSentRelease(order, 'the venue could not be reached for long enough')) anyChanged = true; + } else if (await this.attemptQuarantineCleanup(order, actionIntegration)) { + // see attemptQuarantineCleanup + anyChanged = true; } - // Otherwise — a venue that cannot be asked yet, or an inconclusive answer with nobody having - // released the order — nothing changes and it keeps blocking. + // Otherwise — an order still inside the window in which its request could be live — nothing changes + // and it keeps blocking. } catch (e) { // a failing lookup must never promote the order out of quarantine this.logger.error(`Error in resolving uncertain liquidity order ${order.id}:`, e); @@ -413,13 +493,54 @@ export class LiquidityManagementPipelineService { return anyChanged; } + /** + * Attempt cancel-and-abandon for a quarantined order that has outlived its bound. + * + * Called from two places that share the same exit and must not invent two clocks for it: the ordinary + * end of the if/else-if chain (inconclusive lookup, bound reached), and the SENT branch when the venue + * confirmed the reference but no registered command remains to finish the job. The deadline check lives + * here — not at either call site — so both paths apply the same bound, and so the SENT path can invoke + * cleanup without re-entering the chain or falling through into completeNotSentRelease. + * + * Returns whether the order was abandoned (and thus whether the caller should count a state change). + */ + private async attemptQuarantineCleanup( + order: LiquidityManagementOrder, + actionIntegration: LiquidityActionIntegration, + ): Promise { + if (!order.unresolvableTooLong()) return false; + + // Old enough that cleaning it up is worth attempting, and nobody has released it. Leaving it here + // forever is not the careful option — the rule then never runs again and the venue stops being + // served entirely. Trade and withdraw both reach this path: the integration decides what settles the + // question (cancel every reference, or — for a withdrawal, which cannot be cancelled — a venue reply + // that does not name the reference) and returns the reason wording the abandon step will record. + // + // What stands in the way of giving up is never the order itself but the possibility of a request + // still executing: hand the funds back and a late fill spends them twice. So rather than + // estimating when that can no longer happen — these are orders nothing expires, so age proves + // nothing — the possibility is removed. Cancelling is the opposite of re-sending and cannot create + // anything, and once the venue confirms nothing can execute, abandoning is a fact rather than a + // guess. Refuses to settle, or cannot be reached? Then nothing changes and the order waits on the + // venue (an operator is only a shortcut past the next automatic pass). + const because = await actionIntegration.cancelOutstanding?.(order); + if (!because) return false; + + return this.abandonUncertainOrder(order, because); + } + /** * Put a not-sent conclusion into effect: the order becomes an ordinary failure and its rule may plan anew. * - * The only place an order leaves quarantine downwards. Everything that reaches here has either a venue + * The evidence-based way out of quarantine downwards. Everything that reaches here has either a venue * verdict behind it, or a person who checked plus a venue that has no record — never a single judgement on * its own. The two exceptions are about liveness, not evidence: a venue nothing can ask, and one that has * answered nothing for long enough. Silence there stops vetoing the person who checked; it proves nothing. + * + * The other way out is {@link abandonUncertainOrder}, which concludes nothing about the send itself and + * rests instead on the venue confirming that nothing can still execute — so that an order nobody releases + * is not held by that alone. It is not a guarantee against blocking: where the venue will not confirm, or + * cannot be asked to cancel, this release stays the only way out. */ private async completeNotSentRelease(order: LiquidityManagementOrder, because: string): Promise { // The release this pass looked at, captured before the entity is mutated. Ending an order is the one @@ -436,6 +557,44 @@ export class LiquidityManagementPipelineService { return true; } + /** + * Abandon an order with nothing left outstanding at the venue, so its rule runs again. + * + * The way out of quarantine that rests on no conclusion about whether the request was ever sent — that + * stays unknown, which is why `because` may only ever describe what the venue confirmed, never that + * nothing was sent. The row must not claim an observation nobody made. + * + * The clock does not release anything on its own: it only decides when cleaning up is worth attempting. + * What permits the release is the venue confirming that none of the order's references can execute. + * + * Reached only after the venue has confirmed that none of the order's references can execute any more, so + * what it ends is a wait, not an open question. + * + * Logged as a warning, not an info. Nothing here is routine — an order reaching this point means the venue + * lost track of a request past its bound — and the entry is what makes that visible without an operator + * having to be the mechanism that unblocks it. + */ + private async abandonUncertainOrder(order: LiquidityManagementOrder, because: string): Promise { + // Usually null, because a pending release is handled by an earlier branch for every answer that concerns + // it — but not always: this branch has no release condition of its own, so an order released while the + // venue was unreachable reaches it with the marker still set, and is then given up on the cancellation + // rather than on the release. Whichever it is, the value read here is the one narrowed on, which is the + // point: an operator may write a release between that read and this write, and that release carries an + // audited reason and is owed one more venue answer. Without the narrowing this write — which rests on the + // venue's cancellation but on no evidence about whether the request was ever sent — would silently + // overwrite the one resting on a person. (The reason itself survives either way: the abandonment prefixes + // the existing message rather than replacing it.) + const examined = order.notSentRecheckDue ?? null; + + order.abandonUncertain(`${order.errorMessage} (abandoned ${new Date().toISOString()}: ${because})`); + + if (!(await this.leaveQuarantine(order, examined))) return false; + + this.logger.warn(`Uncertain liquidity order ${order.id} abandoned: ${because}`); + + return true; + } + /** * Record that the venue holds this order — and make sure that fact lands somewhere durable. * @@ -576,8 +735,16 @@ export class LiquidityManagementPipelineService { { id: order.id, status: LiquidityManagementOrderStatus.UNCERTAIN, - // narrowed by the caller when the outcome depends on WHICH pending release was examined - ...(expectedRecheckDue !== undefined ? { notSentRecheckDue: expectedRecheckDue } : {}), + // Narrowed by the caller when the outcome depends on WHICH pending release was examined. + // + // `IsNull()` rather than a bare null: TypeORM renders a raw null in a where object as `= NULL` + // (invalidWhereValuesBehavior.null defaults to "ignore", which falls through to an equality), and + // `x = NULL` is UNKNOWN in SQL, so it matches nothing — not even the row whose column really is + // NULL. "No release was pending" is the ordinary case for every caller here, so without this the + // narrowed update would silently never affect a row and report itself as a lost race. + ...(expectedRecheckDue !== undefined + ? { notSentRecheckDue: expectedRecheckDue === null ? IsNull() : expectedRecheckDue } + : {}), }, { status: order.status, @@ -591,7 +758,21 @@ export class LiquidityManagementPipelineService { ); if (!result.affected) { - this.logger.info(`Uncertain liquidity order ${order.id} was already resolved elsewhere, skipping`); + // Two very different situations, and this write cannot tell them apart: either somebody resolved the + // order between the read and here — a race, which the next pass simply sees — or the narrowing above + // matched no row and never will, in which case this repeats on every pass while the order stays exactly + // as it is. Claiming the benign one was wrong: a release timestamp written with microsecond precision + // cannot be matched by a JavaScript Date, which carries milliseconds, and one written by hand in SQL held + // a live order for hours behind the reassuring version of this line. + // + // So it names both and says what to look for. A warning rather than info because a race is rare and + // self-correcting while the other case is a silent permanent block, and this line is the only trace it + // leaves — the whole failure this branch exists to end. + this.logger.warn( + `Uncertain liquidity order ${order.id} was not updated: either it was resolved elsewhere, or the ` + + `quarantine narrowing matched no row. Repeating on every pass means the latter — the order is stuck.`, + ); + return false; } @@ -628,9 +809,12 @@ export class LiquidityManagementPipelineService { * Release a quarantined order by hand, after somebody checked the venue directly. * * Reconciliation can only ever confirm that a reference exists; it never concludes the opposite, because - * no venue reply proves "this was never accepted". Without this path a genuinely unsent request would - * block its rule forever. Guarded like the payout subdomain's retry: the caller must assert the check and - * name where it happened, and the assertion is recorded on the order. + * no venue reply proves "this was never accepted". An order that can at least be cancelled is given up + * once the venue confirms nothing can still execute, so this path is what keeps the rest moving: orders no + * integration can look up, and venues that will not settle them, which nothing here would otherwise + * release. Guarded like the + * payout subdomain's retry: the caller must assert the check and name where it happened, and the + * assertion is recorded on the order. */ async resolveUncertainOrderManually( orderId: number, @@ -737,7 +921,20 @@ export class LiquidityManagementPipelineService { } private async checkOrder(order: LiquidityManagementOrder): Promise { + // A running order whose command is no longer registered cannot be completed through the normal path. + // getIntegration returns null for unregistered commands; without this guard the next line would throw a + // TypeError that checkRunningOrders only logs — the order would stay IN_PROGRESS forever, outside + // quarantine, where neither automatic abandon nor the manual release endpoint can reach it. Quarantining + // via OrderOutcomeUnknownException is not a Scrypt special case: for any system, a live order without an + // adapter is better in UNCERTAIN (automatic and manual exits both apply) than in a state where nothing + // acts on it. const actionIntegration = this.actionIntegrationFactory.getIntegration(order.action); + if (!actionIntegration) { + throw new OrderOutcomeUnknownException( + `Liquidity order ${order.id} has no registered integration for ${order.action.system}/${order.action.command} that can check its completion`, + ); + } + const isComplete = await actionIntegration.checkCompletion(order); if (isComplete) { diff --git a/src/subdomains/generic/kyc/services/__tests__/kyc.service.spec.ts b/src/subdomains/generic/kyc/services/__tests__/kyc.service.spec.ts index 73d6fdacdc..582086c6aa 100644 --- a/src/subdomains/generic/kyc/services/__tests__/kyc.service.spec.ts +++ b/src/subdomains/generic/kyc/services/__tests__/kyc.service.spec.ts @@ -3,6 +3,7 @@ import { ForbiddenException } from '@nestjs/common'; import { Configuration, ConfigService } from 'src/config/config'; import { BlobContent } from 'src/integration/infrastructure/storage/storage.service'; import { JwtPayload } from 'src/shared/auth/jwt-payload.interface'; +import { SetStaffKycClearance } from 'src/shared/auth/staff-kyc-clearance'; import { UserRole } from 'src/shared/auth/user-role.enum'; import { createCustomCountry } from 'src/shared/models/country/__mocks__/country.entity.mock'; import { Country } from 'src/shared/models/country/country.entity'; @@ -120,6 +121,10 @@ describe('KycService getFileByUid protected-file access', () => { }); beforeEach(() => { + // Protected-file access now additionally requires staff KYC clearance for the calling account + // (account 1 in `jwtFor`); the uncleared case has its own test below. + SetStaffKycClearance([1]); + kycFileService = createMock(); documentService = createMock(); tfaService = { check: jest.fn() }; @@ -164,6 +169,28 @@ describe('KycService getFileByUid protected-file access', () => { expect(documentService.downloadFile).not.toHaveBeenCalled(); }); + // This route is OptionalJwtAuthGuard-only, so no RoleGuard has applied the staff KYC gate — the role + // check inside the service is the only thing standing between an uncleared admin and the most + // sensitive sink in the API. + describe.each([UserRole.SUPER_ADMIN, UserRole.ADMIN, UserRole.COMPLIANCE])('%s without KYC clearance', (role) => { + it('is forbidden from a protected file, without downloading', async () => { + SetStaffKycClearance([]); + kycFileService.getKycFile.mockResolvedValue(kycFile()); + + await expect(service.getFileByUid('FILE-UID', jwtFor(role), ip)).rejects.toBeInstanceOf(ForbiddenException); + expect(documentService.downloadFile).not.toHaveBeenCalled(); + }); + + it('still serves a non-protected file', async () => { + SetStaffKycClearance([]); + kycFileService.getKycFile.mockResolvedValue(kycFile({ protected: false })); + + const dto = await service.getFileByUid('FILE-UID', jwtFor(role), ip); + + expect(dto.uid).toBe('FILE-UID'); + }); + }); + // a blocked account keeps its JWT role until expiry, so the status check must still deny access describe.each<[UserRole, Partial]>([ [UserRole.ADMIN, { accountStatus: UserDataStatus.BLOCKED }], diff --git a/src/subdomains/generic/kyc/services/kyc.service.ts b/src/subdomains/generic/kyc/services/kyc.service.ts index 5f6cb2dda9..92de4fa10f 100644 --- a/src/subdomains/generic/kyc/services/kyc.service.ts +++ b/src/subdomains/generic/kyc/services/kyc.service.ts @@ -9,7 +9,9 @@ import { import { CronExpression } from '@nestjs/schedule'; import { Config } from 'src/config/config'; import { JwtPayload } from 'src/shared/auth/jwt-payload.interface'; +import { StaffKycRequiredException } from 'src/shared/auth/exceptions/staff-kyc-required.exception'; import { hasRoleAccess } from 'src/shared/auth/role.guard'; +import { HasStaffKycClearance } from 'src/shared/auth/staff-kyc-clearance'; import { isUserActive } from 'src/shared/auth/user-active.guard'; import { UserRole } from 'src/shared/auth/user-role.enum'; import { Country } from 'src/shared/models/country/country.entity'; @@ -473,8 +475,14 @@ export class KycService { if (!kycFile) throw new NotFoundException('KYC file not found'); if (kycFile.protected) { + // This route is OptionalJwtAuthGuard-only, so no RoleGuard has applied the staff KYC gate. Protected + // KYC files are the most sensitive sink in the API — an uncleared compliance/admin account must not + // reach them just because the endpoint is not role-gated. The two conditions are checked separately + // so the caller learns which one failed: "wrong role" and "role fine, identification missing" need + // different actions, and a single message for both sends staff looking in the wrong place. if (!hasRoleAccess(UserRole.COMPLIANCE, jwt?.role)) throw new ForbiddenException('Requires admin or compliance role'); + if (!HasStaffKycClearance(jwt?.account)) throw new StaffKycRequiredException(); if (!jwt || !isUserActive(jwt)) throw new ForbiddenException('User is not active'); // Mail-origin staff sessions (tfaRequired) must complete STRICT 2FA before downloading protected KYC diff --git a/src/subdomains/generic/user/models/user/__tests__/staff-kyc-clearance.pg.spec.ts b/src/subdomains/generic/user/models/user/__tests__/staff-kyc-clearance.pg.spec.ts new file mode 100644 index 0000000000..79ad9d2f4a --- /dev/null +++ b/src/subdomains/generic/user/models/user/__tests__/staff-kyc-clearance.pg.spec.ts @@ -0,0 +1,118 @@ +import { DataSource } from 'typeorm'; +import { BlankChars, nonBlankPredicate } from '../staff-kyc-clearance.service'; + +// The clearance query decides who reaches every elevated endpoint, and its verifiedName condition is raw +// SQL: a mocked repository never executes it, so the unit suite cannot tell a correct predicate from one +// that silently clears blank names. This suite runs the exact fragment against a real Postgres and pins +// it to the semantics it replaced — `String.prototype.trim()`. +// +// Skipped unless a connection string is provided, matching the migration suites in this repo; CI runs it +// against its throwaway Postgres service. +const PG_URL = process.env.MIGRATION_TEST_PG; +const describeDb = PG_URL ? describe : describe.skip; + +const SCHEMA = 'staff_kyc_clearance_spec'; + +// Blank in JS, and every one of them a plausible copy-paste artefact in an identity field. Bare +// Postgres `TRIM()` strips only U+0020, so all but the plain-space cases would survive it. +const BLANK_NAMES = [ + '', + '\u0020', + '\u0020\u0020\u0020', + '\u0009', // tab + '\u000a', // line feed + '\u000d\u000a', // CRLF + '\u000b', // vertical tab + '\u000c', // form feed + '\u00a0', // non-breaking space + '\u1680', // ogham space mark + '\u2003', // em space + '\u202f', // narrow no-break space + '\u205f', // medium mathematical space + '\u3000', // ideographic space + '\ufeff', // zero-width no-break space + '\u0020\u0009\u00a0\u3000\u0020', // mixed +]; + +// Every character the JS runtime treats as blank, derived from the runtime rather than from BlankChars: +// fixtures generated from the constant under test would shrink along with it and could never catch a +// character dropped from it. +const JS_BLANK_CHARS = Array.from({ length: 0x10000 }, (_, code) => String.fromCharCode(code)).filter( + (char) => char.trim() === '', +); + +// Real identifications, including ones padded with the same characters — padding must never disqualify. +const REAL_NAMES = [ + 'Alice Example', + '\u0020Alice Example\u0020', + '\u0009Alice Example\u000a', + '\u00a0Alice Example\u3000', + 'X', +]; + +describeDb('staff KYC clearance verifiedName predicate (real Postgres)', () => { + let dataSource: DataSource; + + beforeAll(async () => { + dataSource = new DataSource({ type: 'postgres', url: PG_URL }); + await dataSource.initialize(); + await dataSource.query(`DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE`); + await dataSource.query(`CREATE SCHEMA ${SCHEMA}`); + // Quoted camelCase column, exactly as the migrations create it — an unquoted identifier would be + // folded to lowercase by Postgres and the predicate would fail at runtime rather than in review. + await dataSource.query(`CREATE TABLE ${SCHEMA}.user_data (id serial PRIMARY KEY, "verifiedName" varchar)`); + }); + + afterAll(async () => { + if (!dataSource?.isInitialized) return; + await dataSource.query(`DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE`); + await dataSource.destroy(); + }); + + async function selectMatching(names: (string | null)[]): Promise<(string | null)[]> { + await dataSource.query(`TRUNCATE ${SCHEMA}.user_data`); + for (const name of names) { + await dataSource.query(`INSERT INTO ${SCHEMA}.user_data ("verifiedName") VALUES ($1)`, [name]); + } + + // $1 stands in for the named :blankChars parameter TypeORM binds; the fragment itself is verbatim. + const predicate = nonBlankPredicate('"verifiedName"').replace(':blankChars', '$1'); + const rows = await dataSource.query( + `SELECT "verifiedName" FROM ${SCHEMA}.user_data WHERE ${predicate} ORDER BY id`, + [BlankChars], + ); + + return rows.map((row: { verifiedName: string | null }) => row.verifiedName); + } + + it('excludes NULL', async () => { + await expect(selectMatching([null])).resolves.toEqual([]); + }); + + it.each(BLANK_NAMES)('excludes the blank name %j', async (name) => { + await expect(selectMatching([name])).resolves.toEqual([]); + }); + + it.each(REAL_NAMES)('keeps the real name %j', async (name) => { + await expect(selectMatching([name])).resolves.toEqual([name]); + }); + + // Guards the character set itself: drop a character from BlankChars and a name consisting of it starts + // clearing an account. The named cases above stay for readability — this one is the exhaustive check. + it('excludes every character the JS runtime treats as blank', async () => { + expect(JS_BLANK_CHARS.length).toBeGreaterThan(20); // sanity: the derivation actually found them + + await expect(selectMatching(JS_BLANK_CHARS)).resolves.toEqual([]); + }); + + // The property that matters: the SQL predicate and the JS check it replaced must agree on every input. + // A disagreement here is either a locked-out staff member or a cleared account with no identification. + it('agrees with String.prototype.trim() on every case', async () => { + const all = [null, ...BLANK_NAMES, ...JS_BLANK_CHARS, ...REAL_NAMES]; + + const fromSql = await selectMatching(all); + const fromJs = all.filter((name) => name?.trim()); + + expect(fromSql).toEqual(fromJs); + }); +}); diff --git a/src/subdomains/generic/user/models/user/__tests__/staff-kyc-clearance.service.spec.ts b/src/subdomains/generic/user/models/user/__tests__/staff-kyc-clearance.service.spec.ts new file mode 100644 index 0000000000..6b95fabb01 --- /dev/null +++ b/src/subdomains/generic/user/models/user/__tests__/staff-kyc-clearance.service.spec.ts @@ -0,0 +1,103 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { SettingService } from 'src/shared/models/setting/setting.service'; +import { StaffKycClearanceService } from '../staff-kyc-clearance.service'; +import { UserRepository } from '../user.repository'; + +describe('StaffKycClearanceService', () => { + let service: StaffKycClearanceService; + let userRepo: UserRepository; + let settingService: SettingService; + + function setup(users: unknown[]): void { + jest.spyOn(userRepo, 'find').mockResolvedValue(users as never); + } + + // Rows as the DB returns them: the role / kycLevel / verifiedName filtering has already happened in + // SQL, so a returned row is by definition a cleared one. + function staffUser(accountId: number): unknown { + return { id: accountId * 10, userData: { id: accountId } }; + } + + beforeEach(async () => { + userRepo = { find: jest.fn() } as unknown as UserRepository; + settingService = { setObj: jest.fn() } as unknown as SettingService; + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + StaffKycClearanceService, + { provide: UserRepository, useValue: userRepo }, + { provide: SettingService, useValue: settingService }, + ], + }).compile(); + + service = module.get(StaffKycClearanceService); + }); + + afterEach(() => jest.resetAllMocks()); + + it('writes the cleared account ids to the staffKycClearance setting', async () => { + setup([staffUser(11), staffUser(12)]); + + await service.syncStaffKycClearance(); + + expect(settingService.setObj).toHaveBeenCalledWith('staffKycClearance', [11, 12]); + }); + + // Accounts without a usable verifiedName (NULL, empty or whitespace-only) are excluded by the SQL + // predicate, not in JS — so the assertion has to be on the query. `TRIM(...) <> ''` also drops NULL, + // because the comparison yields NULL rather than true. + it('excludes names that are NULL, empty or whitespace-only via a SQL predicate', async () => { + setup([]); + + await service.syncStaffKycClearance(); + + const verifiedName = (userRepo.find as jest.Mock).mock.calls[0][0].where.userData.verifiedName; + expect(verifiedName.type).toBe('raw'); + // The alias must be interpolated verbatim: TypeORM passes it in already quoted, and on Postgres an + // unquoted camelCase identifier would be folded to lowercase and blow up at runtime. + expect(verifiedName.getSql('"UserData"."verifiedName"')).toBe( + `BTRIM("UserData"."verifiedName", :blankChars) <> ''`, + ); + // Bare TRIM() would strip ASCII space only; the bound set must carry the characters that make the + // predicate agree with String.prototype.trim(). Whether it actually does is proven against a real + // database in staff-kyc-clearance.pg.spec.ts — a mocked repository never runs the fragment. + expect(verifiedName.objectLiteralParameters.blankChars).toContain('\u0009'); + expect(verifiedName.objectLiteralParameters.blankChars).toContain('\u00a0'); + expect(verifiedName.objectLiteralParameters.blankChars).toContain('\u3000'); + }); + + it('deduplicates accounts backing several staff users', async () => { + // One person can hold multiple staff wallets pointing at the same user data. + setup([staffUser(11), staffUser(11)]); + + await service.syncStaffKycClearance(); + + expect(settingService.setObj).toHaveBeenCalledWith('staffKycClearance', [11]); + }); + + it('writes an empty list when nobody qualifies — the gate is fail-closed', async () => { + setup([]); + + await service.syncStaffKycClearance(); + + expect(settingService.setObj).toHaveBeenCalledWith('staffKycClearance', []); + }); + + it('queries only staff roles and kycLevel >= 50', async () => { + setup([]); + + await service.syncStaffKycClearance(); + + const where = (userRepo.find as jest.Mock).mock.calls[0][0].where; + expect(where.role._value).toEqual(expect.arrayContaining(['Admin', 'SuperAdmin', 'Debug', 'RealUnit'])); + expect(where.role._value).not.toContain('User'); + expect(where.userData.kycLevel._value).toBe(50); + }); + + it('does not swallow a repository failure — a failed sync must keep the last known Set', async () => { + jest.spyOn(userRepo, 'find').mockRejectedValue(new Error('db down')); + + await expect(service.syncStaffKycClearance()).rejects.toThrow('db down'); + expect(settingService.setObj).not.toHaveBeenCalled(); + }); +}); diff --git a/src/subdomains/generic/user/models/user/staff-kyc-clearance.service.ts b/src/subdomains/generic/user/models/user/staff-kyc-clearance.service.ts new file mode 100644 index 0000000000..2a3aa9feb6 --- /dev/null +++ b/src/subdomains/generic/user/models/user/staff-kyc-clearance.service.ts @@ -0,0 +1,74 @@ +import { Injectable } from '@nestjs/common'; +import { CronExpression } from '@nestjs/schedule'; +import { rolesSatisfying } from 'src/shared/auth/role.guard'; +import { KycGatedRoles } from 'src/shared/auth/user-role.enum'; +import { SettingService } from 'src/shared/models/setting/setting.service'; +import { DfxCron } from 'src/shared/utils/cron'; +import { In, MoreThanOrEqual, Raw } from 'typeorm'; +import { KycLevel } from '../user-data/user-data.enum'; +import { UserRepository } from './user.repository'; + +// Roles that can reach a KYC-gated endpoint: the gated entry roles plus their super-roles (e.g. +// SUPER_ADMIN, which satisfies every gate but is not itself listed in KycGatedRoles). Derived, not +// hand-written — a role added to the hierarchy must not silently fall out of the clearance sync and +// lose access. +const ClearanceRelevantRoles = rolesSatisfying(KycGatedRoles); + +// Every character `String.prototype.trim()` strips (ECMAScript WhiteSpace + LineTerminator). Postgres' +// bare `TRIM(x)` removes ASCII space ONLY, so a name of a single tab or a non-breaking space would pass +// a `TRIM(x) <> ''` test and clear an account that carries no identification at all. The set is spelled +// out rather than left to `[[:space:]]`, whose meaning depends on the database locale. +// +// U+200B (zero width space) is deliberately NOT in here: `trim()` does not strip it either, so adding it +// would make this predicate stricter than the check it replaced. Whether a name made of invisible +// characters that `trim()` ignores should count as identification is a question about how verifiedName is +// written, not about this gate. +export const BlankChars = + '\u0009\u000a\u000b\u000c\u000d\u0020\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007' + + '\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff'; + +// Exported so the Postgres suite can execute this exact predicate against a real database — a mocked +// repository never runs the fragment, and the ASCII-only default of `TRIM` is invisible without one. +export function nonBlankPredicate(alias: string): string { + return `BTRIM(${alias}, :blankChars) <> ''`; +} + +// Maintains the `staffKycClearance` setting: the account (user data) ids allowed onto elevated +// endpoints. `ProcessService` primes the in-memory Set from it and `RoleGuard` enforces it — see +// `HasStaffKycClearance` for the fail-closed semantics. +// +// Mirrors JwtRevocationSyncService: the cron lives in the user domain (which owns User/UserData) to +// keep the shared ProcessService/SettingService free of subdomain dependencies. Self-healing in both +// directions — losing kycLevel, losing verifiedName, or losing the staff role drops the account out of +// the query and thus out of the setting on the next run. +@Injectable() +export class StaffKycClearanceService { + constructor( + private readonly userRepo: UserRepository, + private readonly settingService: SettingService, + ) {} + + // Every minute, matching JwtRevocationSyncService: revoking elevated access promptly is a security + // requirement and warrants the same exception to the "prefer 15min" cron guideline. + @DfxCron(CronExpression.EVERY_MINUTE, { timeout: 1800 }) + async syncStaffKycClearance(): Promise { + const staffUsers = await this.userRepo.find({ + select: { id: true, userData: { id: true } }, + where: { + role: In(ClearanceRelevantRoles), + userData: { + kycLevel: MoreThanOrEqual(KycLevel.LEVEL_50), + // `verifiedName IS NOT NULL` is the stated rule, but an empty or blank name carries no + // identification either — the predicate covers both, and NULL drops out on its own because the + // comparison yields NULL. See BlankChars for why the character set is explicit. + verifiedName: Raw(nonBlankPredicate, { blankChars: BlankChars }), + }, + }, + relations: { userData: true }, + }); + + const clearedAccounts = staffUsers.map((user) => user.userData.id); + + await this.settingService.setObj('staffKycClearance', [...new Set(clearedAccounts)]); + } +} diff --git a/src/subdomains/generic/user/user.module.ts b/src/subdomains/generic/user/user.module.ts index 5fce80a978..d6d0860a7c 100644 --- a/src/subdomains/generic/user/user.module.ts +++ b/src/subdomains/generic/user/user.module.ts @@ -48,6 +48,7 @@ import { JwtRevocationSyncService } from './models/user-data/jwt-revocation-sync import { UserDataJobService } from './models/user-data/user-data-job.service'; import { UserDataNotificationService } from './models/user-data/user-data-notification.service'; import { UserData } from './models/user-data/user-data.entity'; +import { StaffKycClearanceService } from './models/user/staff-kyc-clearance.service'; import { UserJobService } from './models/user/user-job.service'; import { UserController, UserV2Controller } from './models/user/user.controller'; import { User } from './models/user/user.entity'; @@ -126,6 +127,7 @@ import { WebhookService } from './services/webhook/webhook.service'; OrganizationRepository, UserDataJobService, JwtRevocationSyncService, + StaffKycClearanceService, UserJobService, RecommendationRepository, RecommendationService, diff --git a/src/subdomains/supporting/support-issue/__tests__/support-issue.controller.spec.ts b/src/subdomains/supporting/support-issue/__tests__/support-issue.controller.spec.ts index ddb38bddf6..f95f40e168 100644 --- a/src/subdomains/supporting/support-issue/__tests__/support-issue.controller.spec.ts +++ b/src/subdomains/supporting/support-issue/__tests__/support-issue.controller.spec.ts @@ -1,6 +1,7 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest'; import { ModuleRef } from '@nestjs/core'; import { JwtPayload } from 'src/shared/auth/jwt-payload.interface'; +import { SetStaffKycClearance } from 'src/shared/auth/staff-kyc-clearance'; import { UserRole } from 'src/shared/auth/user-role.enum'; import { CreateSupportMessageDto } from '../dto/create-support-message.dto'; import { SupportEscalationService } from '../services/support-escalation.service'; @@ -28,6 +29,10 @@ describe('SupportIssueController.createSupportMessage routing', () => { const ip = '1.2.3.4'; beforeEach(() => { + // Staff routing now additionally requires KYC clearance for the calling account (account 7 in the + // staff cases below); the uncleared case has its own test. + SetStaffKycClearance([7]); + service = createMock(); tfaService = { check: jest.fn() }; moduleRef = createMock(); @@ -47,6 +52,22 @@ describe('SupportIssueController.createSupportMessage routing', () => { }, ); + // This route is OptionalJwtAuthGuard-only, so the staff KYC gate has to be applied inline. An + // uncleared staff account falls through to the customer path rather than posting an official reply. + describe.each([UserRole.SUPPORT, UserRole.COMPLIANCE, UserRole.ADMIN, UserRole.SUPER_ADMIN])( + 'staff role %s without KYC clearance', + (role) => { + it('falls through to createMessage instead of posting an official reply', async () => { + SetStaffKycClearance([]); + + await controller.createSupportMessage({ role, account: 7 } as JwtPayload, '42', dto, ip); + + expect(service.createMessageSupport).not.toHaveBeenCalled(); + expect(service.createMessage).toHaveBeenCalledWith('42', dto, 7); + }); + }, + ); + it('routes a regular user message to createMessage', async () => { await controller.createSupportMessage({ role: UserRole.USER, account: 7 } as JwtPayload, '42', dto, ip); diff --git a/src/subdomains/supporting/support-issue/support-issue.controller.ts b/src/subdomains/supporting/support-issue/support-issue.controller.ts index a36d64077c..9a36160e8b 100644 --- a/src/subdomains/supporting/support-issue/support-issue.controller.ts +++ b/src/subdomains/supporting/support-issue/support-issue.controller.ts @@ -7,7 +7,7 @@ import { GetJwt } from 'src/shared/auth/get-jwt.decorator'; import { JwtPayload } from 'src/shared/auth/jwt-payload.interface'; import { OptionalJwtAuthGuard } from 'src/shared/auth/optional.guard'; import { RealIP } from 'src/shared/auth/real-ip.decorator'; -import { hasRoleAccess, RoleGuard } from 'src/shared/auth/role.guard'; +import { hasStaffAccess, RoleGuard } from 'src/shared/auth/role.guard'; import { isUserActive, UserActiveGuard } from 'src/shared/auth/user-active.guard'; import { UserRole } from 'src/shared/auth/user-role.enum'; import { CLIENT_HEADER } from 'src/shared/utils/request-client'; @@ -205,7 +205,9 @@ export class SupportIssueController { // Staff routing requires an active account: blocked staff keep their JWT role until token // expiry (default 2d) but must not be able to post official replies. Non-staff callers // (including anonymous, since the guard is Optional) fall through to createMessage. - if (jwt?.role && hasRoleAccess(UserRole.SUPPORT, jwt.role) && isUserActive(jwt)) { + // `hasStaffAccess`, not `hasRoleAccess`: this route is OptionalJwtAuthGuard-only, so the staff KYC gate + // has to be applied here — posting official DFX replies is a staff privilege like any other. + if (jwt?.role && hasStaffAccess(UserRole.SUPPORT, jwt) && isUserActive(jwt)) { // Mail-origin staff sessions must complete STRICT 2FA before posting an official reply. The global // TfaEnforcementInterceptor already enforces this invariant on every route; this inline check is kept as // defense-in-depth on this sensitive sink. Wallet-signature logins (no tfaRequired) are unaffected.