From 4fc24d5ee022151195765b25942992a5a0de6f22 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:02 +0200 Subject: [PATCH 01/86] Let an instance serve HTTP without registering cron jobs Background jobs and HTTP requests share a single Node event loop, and Node runs JavaScript on one thread. A CPU-heavy job therefore delays every request on the same instance: measured on production, a request to /version - no database access, handler time 1 ms - took p95 5.5 s when measured locally against the container, with event loop utilization averaging 84%. The delay followed the cron jitter window exactly, so it was driven by the scheduler, not by traffic. CRON_JOBS_ENABLED=false now makes DfxCronService return from onModuleInit without registering anything, which allows running the jobs in a second instance while the first only answers HTTP. The switch is deliberately coarser than DISABLED_PROCESSES: that one only skips jobs declaring a `process`, and 21 of 131 jobs declare none - among them trade checks, referral credits and volume resets. Those would keep running on both instances, and since cron locks are per-process, nothing would catch the duplicate execution. An unset variable keeps the previous behaviour so every existing environment is unaffected. Any value other than 'true'/'false' throws instead of being coerced: a typo such as 'fals' would otherwise silently re-enable the scheduler on an instance meant to be HTTP-only, and duplicate execution of financial jobs is far more damaging than a failed boot. --- .../cron-jobs-enabled.config.spec.ts | 24 ++++++ src/config/config.ts | 25 ++++++ .../__tests__/dfx-cron.service.spec.ts | 80 +++++++++++++++++++ src/shared/services/dfx-cron.service.ts | 8 ++ 4 files changed, 137 insertions(+) create mode 100644 src/config/__tests__/cron-jobs-enabled.config.spec.ts create mode 100644 src/shared/services/__tests__/dfx-cron.service.spec.ts diff --git a/src/config/__tests__/cron-jobs-enabled.config.spec.ts b/src/config/__tests__/cron-jobs-enabled.config.spec.ts new file mode 100644 index 0000000000..24350a959e --- /dev/null +++ b/src/config/__tests__/cron-jobs-enabled.config.spec.ts @@ -0,0 +1,24 @@ +import { parseCronJobsEnabled } from '../config'; + +describe('parseCronJobsEnabled', () => { + it('defaults to enabled when unset, so existing environments keep running jobs', () => { + expect(parseCronJobsEnabled(undefined)).toBe(true); + expect(parseCronJobsEnabled('')).toBe(true); + }); + + it('accepts the two valid values', () => { + expect(parseCronJobsEnabled('true')).toBe(true); + expect(parseCronJobsEnabled('false')).toBe(false); + }); + + it.each(['fals', 'False', 'FALSE', '0', 'no', 'off', 'disabled', ' false'])( + 'throws on %p instead of silently enabling the scheduler', + (value) => { + // The dangerous direction is a typo being read as "enabled": on an instance meant to be + // HTTP-only that would re-register every job, and jobs without a `process` (trades, + // referral credits, volume resets) would then run on two instances at once. Cron locks + // are per-process, so nothing else would catch it. Failing the boot is the safe outcome. + expect(() => parseCronJobsEnabled(value)).toThrow(/expected 'true' or 'false'/); + }, + ); +}); diff --git a/src/config/config.ts b/src/config/config.ts index 785a112b1d..e677685e35 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -1496,6 +1496,15 @@ export class Configuration { return splitWithdrawKeys(process.env.EVM_WALLETS); } + // Background jobs and HTTP requests share a single Node event loop, so a busy scheduler + // delays every incoming request on the same instance. Setting this to false keeps an + // instance HTTP-only: DfxCronService then registers no job at all. + // + // Note this is deliberately independent of DISABLED_PROCESSES, which only skips jobs that + // declare a `process` — jobs without one would keep running and, on a second instance, + // run twice. Cron locks are per-process and do not guard across instances. + cronJobsEnabled = parseCronJobsEnabled(process.env.CRON_JOBS_ENABLED); + // --- HELPERS --- // disabledProcesses = () => process.env.DISABLED_PROCESSES === '*' @@ -1503,6 +1512,22 @@ export class Configuration { : ((process.env.DISABLED_PROCESSES?.split(',') ?? []) as Process[]); } +/** + * Reads the CRON_JOBS_ENABLED flag. + * + * Unset means "run jobs", which keeps every existing environment working unchanged. Any other + * value than 'true'/'false' throws instead of being coerced: a typo such as 'fals' would + * otherwise silently enable the scheduler on an instance meant to be HTTP-only, and duplicate + * job execution is far more damaging than a failed boot. + */ +export function parseCronJobsEnabled(value?: string): boolean { + if (value == null || value === '') return true; + if (value === 'true') return true; + if (value === 'false') return false; + + throw new Error(`Invalid CRON_JOBS_ENABLED value '${value}': expected 'true' or 'false'`); +} + function readCert(): string | undefined { const path = process.env.LIGHTNING_API_CERTIFICATE_PATH; if (path) { diff --git a/src/shared/services/__tests__/dfx-cron.service.spec.ts b/src/shared/services/__tests__/dfx-cron.service.spec.ts new file mode 100644 index 0000000000..9f7fe92877 --- /dev/null +++ b/src/shared/services/__tests__/dfx-cron.service.spec.ts @@ -0,0 +1,80 @@ +const mockStart = jest.fn(); + +jest.mock('cron', () => ({ + CronJob: jest.fn().mockImplementation(() => ({ start: mockStart })), +})); + +import { createMock } from '@golevelup/ts-jest'; +import { CronExpression, SchedulerRegistry } from '@nestjs/schedule'; +import { DiscoveryService, MetadataScanner } from '@nestjs/core'; +import { Config, ConfigService, GetConfig } from 'src/config/config'; +import { DFX_CRONJOB_PARAMS, DfxCronParams } from 'src/shared/utils/cron'; +import { Process } from '../process.service'; +import { DfxCronService } from '../dfx-cron.service'; + +/** Builds a provider instance carrying @DfxCron metadata, as the decorator would. */ +function providerWithJob(methodName: string, params: DfxCronParams): { instance: object } { + const instance = { + [methodName]: function () { + // no-op job body + }, + }; + + Reflect.defineMetadata(DFX_CRONJOB_PARAMS, params, instance[methodName]); + + return { instance }; +} + +function buildService(providers: { instance: object }[]): { + service: DfxCronService; + scheduler: SchedulerRegistry; +} { + const discovery = createMock({ + getProviders: () => + providers.map((p) => ({ ...p, isDependencyTreeStatic: () => true })) as ReturnType< + DiscoveryService['getProviders'] + >, + }); + const metadataScanner = createMock({ + getAllMethodNames: (instance: object) => Object.keys(instance), + }); + const scheduler = createMock(); + + return { service: new DfxCronService(discovery, metadataScanner, scheduler), scheduler }; +} + +describe('DfxCronService', () => { + const configuredJobs = [ + providerWithJob('withProcess', { expression: CronExpression.EVERY_MINUTE, process: Process.MONITOR_EVENT_LOOP }), + // A job without `process` — DISABLED_PROCESSES cannot stop this one, only the global switch can. + providerWithJob('withoutProcess', { expression: CronExpression.EVERY_MINUTE }), + ]; + + afterEach(() => { + jest.clearAllMocks(); + new ConfigService(GetConfig()); + }); + + it('registers jobs when cron is enabled', () => { + new ConfigService({ ...GetConfig(), cronJobsEnabled: true } as typeof Config); + + const { service, scheduler } = buildService(configuredJobs); + service.onModuleInit(); + + expect(scheduler.addCronJob).toHaveBeenCalledTimes(2); + expect(mockStart).toHaveBeenCalledTimes(2); + }); + + it('registers no job at all when cron is disabled, including jobs without a process', () => { + // The safety property of the HTTP-only instance: were a job without `process` still + // registered here, it would run on both the HTTP and the job instance simultaneously. + // Cron locks are per-process, so duplicate execution would go unnoticed. + new ConfigService({ ...GetConfig(), cronJobsEnabled: false } as typeof Config); + + const { service, scheduler } = buildService(configuredJobs); + service.onModuleInit(); + + expect(scheduler.addCronJob).not.toHaveBeenCalled(); + expect(mockStart).not.toHaveBeenCalled(); + }); +}); diff --git a/src/shared/services/dfx-cron.service.ts b/src/shared/services/dfx-cron.service.ts index af3f63c068..55f62ffe74 100644 --- a/src/shared/services/dfx-cron.service.ts +++ b/src/shared/services/dfx-cron.service.ts @@ -28,6 +28,14 @@ export class DfxCronService implements OnModuleInit { ) {} onModuleInit() { + // HTTP-only instances register nothing at all. Returning here (rather than skipping jobs + // individually) also covers jobs that declare no `process` and would otherwise stay active + // despite DISABLED_PROCESSES — on a second instance they would run twice. + if (!Config.cronJobsEnabled) { + this.logger.info('Cron jobs disabled on this instance (CRON_JOBS_ENABLED=false), registering none'); + return; + } + this.discovery .getProviders() .filter((wrapper) => wrapper.isDependencyTreeStatic()) From f5e4aecd75da41a9ffd06125973b521d799bb5e3 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:06 +0200 Subject: [PATCH 02/86] Export event loop saturation as OTLP metrics Traces answer how long a request took, but not why. A request queued behind a saturated event loop is indistinguishable from one waiting on a slow query, so diagnosing the recent latency issue required taking a V8 CPU profile against the running production process - a one-off snapshot obtained through an intervention in production. MonitorEventLoopService already measures utilization and loop delay correctly, but only writes them to the log, and it runs as a cron job: an HTTP-only instance registers no jobs and would report nothing precisely where saturation matters most. Collection is therefore driven by the OTel metric reader instead of the scheduler, so it runs on every instance. Export reuses the existing OTLP pipeline, so no /metrics endpoint and no additional scrape target are needed. Instrument names follow the Node.js runtime semantic conventions. With OTEL_EXPORTER_OTLP_ENDPOINT unset, no meter is registered and the app boots unchanged, matching how tracing is already handled. --- package-lock.json | 122 +++++++++++++++++++++++--- package.json | 2 + src/__tests__/runtime-metrics.spec.ts | 53 +++++++++++ src/__tests__/tracing.spec.ts | 6 ++ src/main.ts | 1 + src/runtime-metrics.ts | 121 +++++++++++++++++++++++++ src/tracing.ts | 15 ++++ 7 files changed, 308 insertions(+), 12 deletions(-) create mode 100644 src/__tests__/runtime-metrics.spec.ts create mode 100644 src/runtime-metrics.ts diff --git a/package-lock.json b/package-lock.json index b293fef6e1..7f440e6dc2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -44,7 +44,9 @@ "@noble/curves": "^1.9.7", "@opentelemetry/api": "^1.9.1", "@opentelemetry/auto-instrumentations-node": "^0.76.0", + "@opentelemetry/exporter-metrics-otlp-http": "^0.218.0", "@opentelemetry/exporter-trace-otlp-http": "^0.218.0", + "@opentelemetry/sdk-metrics": "^2.10.0", "@opentelemetry/sdk-node": "^0.218.0", "@opentelemetry/sdk-trace-base": "^2.7.1", "@railgun-community/engine": "^9.4.0", @@ -7810,6 +7812,22 @@ "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, + "node_modules/@opentelemetry/exporter-metrics-otlp-grpc/node_modules/@opentelemetry/sdk-metrics": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.7.1.tgz", + "integrity": "sha512-MpDJdkiFDs3Pm1RHO3KByuZbuBdJEXEAkiC0+yJdsZGVCdf1RpHR6n+LHDcS7ffmfrt5kVCzJSCfm4z2C7v0uQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.7.1", + "@opentelemetry/resources": "2.7.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, "node_modules/@opentelemetry/exporter-metrics-otlp-grpc/node_modules/cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", @@ -7898,6 +7916,22 @@ "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, + "node_modules/@opentelemetry/exporter-metrics-otlp-http/node_modules/@opentelemetry/sdk-metrics": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.7.1.tgz", + "integrity": "sha512-MpDJdkiFDs3Pm1RHO3KByuZbuBdJEXEAkiC0+yJdsZGVCdf1RpHR6n+LHDcS7ffmfrt5kVCzJSCfm4z2C7v0uQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.7.1", + "@opentelemetry/resources": "2.7.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, "node_modules/@opentelemetry/exporter-metrics-otlp-proto": { "version": "0.218.0", "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-metrics-otlp-proto/-/exporter-metrics-otlp-proto-0.218.0.tgz", @@ -7949,6 +7983,22 @@ "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, + "node_modules/@opentelemetry/exporter-metrics-otlp-proto/node_modules/@opentelemetry/sdk-metrics": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.7.1.tgz", + "integrity": "sha512-MpDJdkiFDs3Pm1RHO3KByuZbuBdJEXEAkiC0+yJdsZGVCdf1RpHR6n+LHDcS7ffmfrt5kVCzJSCfm4z2C7v0uQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.7.1", + "@opentelemetry/resources": "2.7.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, "node_modules/@opentelemetry/exporter-prometheus": { "version": "0.218.0", "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-prometheus/-/exporter-prometheus-0.218.0.tgz", @@ -7998,6 +8048,22 @@ "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, + "node_modules/@opentelemetry/exporter-prometheus/node_modules/@opentelemetry/sdk-metrics": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.7.1.tgz", + "integrity": "sha512-MpDJdkiFDs3Pm1RHO3KByuZbuBdJEXEAkiC0+yJdsZGVCdf1RpHR6n+LHDcS7ffmfrt5kVCzJSCfm4z2C7v0uQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.7.1", + "@opentelemetry/resources": "2.7.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, "node_modules/@opentelemetry/exporter-trace-otlp-grpc": { "version": "0.218.0", "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-grpc/-/exporter-trace-otlp-grpc-0.218.0.tgz", @@ -11694,6 +11760,22 @@ "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, + "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/sdk-metrics": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.7.1.tgz", + "integrity": "sha512-MpDJdkiFDs3Pm1RHO3KByuZbuBdJEXEAkiC0+yJdsZGVCdf1RpHR6n+LHDcS7ffmfrt5kVCzJSCfm4z2C7v0uQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.7.1", + "@opentelemetry/resources": "2.7.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/sdk-trace-base": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.7.1.tgz", @@ -11941,13 +12023,13 @@ } }, "node_modules/@opentelemetry/sdk-metrics": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.7.1.tgz", - "integrity": "sha512-MpDJdkiFDs3Pm1RHO3KByuZbuBdJEXEAkiC0+yJdsZGVCdf1RpHR6n+LHDcS7ffmfrt5kVCzJSCfm4z2C7v0uQ==", + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.10.0.tgz", + "integrity": "sha512-t6r1VSvXNtSDnPXU1FbZeetJb7yyovHmgu0wRSoftxtE0g2rSNhQZQUy69sRUCL+iioJpX8SN/S6wq6ZtvLySQ==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "2.7.1", - "@opentelemetry/resources": "2.7.1" + "@opentelemetry/core": "2.10.0", + "@opentelemetry/resources": "2.10.0" }, "engines": { "node": "^18.19.0 || >=20.6.0" @@ -11957,9 +12039,9 @@ } }, "node_modules/@opentelemetry/sdk-metrics/node_modules/@opentelemetry/core": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.7.1.tgz", - "integrity": "sha512-QAqIj32AtK6+pEVNG7EOVxHdE06RP+FM5qpiEJ4RtDcFIqKUZHYhl7/7UY5efhwmwNAg7j8QbJVBLxMerc0+gw==", + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.10.0.tgz", + "integrity": "sha512-/wNZ8twnEQQA4HoHu22+vcsdru6pWPWxW+7w+FlxT6Id7PE/WIbZmVKkte+PF72e0F2dnImFeHD2syyE1Mw6MQ==", "license": "Apache-2.0", "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" @@ -11972,12 +12054,12 @@ } }, "node_modules/@opentelemetry/sdk-metrics/node_modules/@opentelemetry/resources": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.7.1.tgz", - "integrity": "sha512-DeT6KKolmC4e/dRQvMQ/RwlnzhaqeiFOXY5ngoOPJ07GgVVKxZOg9EcrNZb5aTzUn+iCrJldAgOfQm1O/QfPAQ==", + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.10.0.tgz", + "integrity": "sha512-q6MMm2zhggzsHVNbabYwut+a6nbuQQe3URUoxaojM/8K1IBfwwPzvxIjNi2/lI1TFe+fMHMW9MWhrtDLEXEnkA==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "2.7.1", + "@opentelemetry/core": "2.10.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "engines": { @@ -12086,6 +12168,22 @@ "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, + "node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/sdk-metrics": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.7.1.tgz", + "integrity": "sha512-MpDJdkiFDs3Pm1RHO3KByuZbuBdJEXEAkiC0+yJdsZGVCdf1RpHR6n+LHDcS7ffmfrt5kVCzJSCfm4z2C7v0uQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.7.1", + "@opentelemetry/resources": "2.7.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, "node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/sdk-trace-base": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.7.1.tgz", diff --git a/package.json b/package.json index c47a2bed43..799d020cf0 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,9 @@ "@noble/curves": "^1.9.7", "@opentelemetry/api": "^1.9.1", "@opentelemetry/auto-instrumentations-node": "^0.76.0", + "@opentelemetry/exporter-metrics-otlp-http": "^0.218.0", "@opentelemetry/exporter-trace-otlp-http": "^0.218.0", + "@opentelemetry/sdk-metrics": "^2.10.0", "@opentelemetry/sdk-node": "^0.218.0", "@opentelemetry/sdk-trace-base": "^2.7.1", "@railgun-community/engine": "^9.4.0", diff --git a/src/__tests__/runtime-metrics.spec.ts b/src/__tests__/runtime-metrics.spec.ts new file mode 100644 index 0000000000..74e6c52cf1 --- /dev/null +++ b/src/__tests__/runtime-metrics.spec.ts @@ -0,0 +1,53 @@ +import { IntervalHistogram } from 'perf_hooks'; +import { toEventLoopSample } from '../runtime-metrics'; + +const NS_PER_S = 1e9; + +function fakeHistogram(values: Partial> & { count: number }): IntervalHistogram { + return { + ...values, + percentile: (p: number) => { + const byPercentile: Record = { + 50: 0.05 * NS_PER_S, + 90: 0.5 * NS_PER_S, + 99: 2 * NS_PER_S, + }; + + return byPercentile[p]; + }, + } as unknown as IntervalHistogram; +} + +describe('toEventLoopSample', () => { + it('converts histogram nanoseconds to seconds', () => { + const histogram = fakeHistogram({ + count: 100, + min: 0.001 * NS_PER_S, + max: 5 * NS_PER_S, + mean: 0.3 * NS_PER_S, + }); + + const sample = toEventLoopSample(histogram, 0.85); + + expect(sample.utilization).toBe(0.85); + expect(sample.delay).toEqual({ min: 0.001, max: 5, mean: 0.3, p50: 0.05, p90: 0.5, p99: 2 }); + }); + + it('reports zeros for an empty histogram instead of the Infinity/0 sentinels Node returns', () => { + // A freshly reset histogram returns min = Infinity and max = 0. Exporting Infinity would + // break the series for every consumer, so an empty window must read as all zeros. + const histogram = fakeHistogram({ count: 0, min: Infinity, max: 0, mean: NaN }); + + const sample = toEventLoopSample(histogram, 0); + + expect(sample.delay).toEqual({ min: 0, max: 0, mean: 0, p50: 0, p90: 0, p99: 0 }); + expect(Object.values(sample.delay).every(Number.isFinite)).toBe(true); + }); + + it('passes utilization through unchanged as a 0..1 ratio', () => { + const histogram = fakeHistogram({ count: 1, min: 0, max: 0, mean: 0 }); + + expect(toEventLoopSample(histogram, 0).utilization).toBe(0); + expect(toEventLoopSample(histogram, 1).utilization).toBe(1); + }); +}); diff --git a/src/__tests__/tracing.spec.ts b/src/__tests__/tracing.spec.ts index 424a36e058..908acbdfc1 100644 --- a/src/__tests__/tracing.spec.ts +++ b/src/__tests__/tracing.spec.ts @@ -9,6 +9,12 @@ jest.mock('@opentelemetry/auto-instrumentations-node', () => ({ jest.mock('@opentelemetry/exporter-trace-otlp-http', () => ({ OTLPTraceExporter: jest.fn(), })); +jest.mock('@opentelemetry/exporter-metrics-otlp-http', () => ({ + OTLPMetricExporter: jest.fn(), +})); +jest.mock('@opentelemetry/sdk-metrics', () => ({ + PeriodicExportingMetricReader: jest.fn(), +})); import { SpanKind, SpanStatusCode } from '@opentelemetry/api'; import { ReadableSpan } from '@opentelemetry/sdk-trace-base'; diff --git a/src/main.ts b/src/main.ts index 96976213a5..744ea6017a 100644 --- a/src/main.ts +++ b/src/main.ts @@ -3,6 +3,7 @@ // the `eventsource` package and ultimately Node's `http`) follows so its // internal HTTP usage is auto-instrumented. import './tracing'; +import './runtime-metrics'; // event loop saturation gauges; must follow ./tracing (needs its meter provider) import './polyfills'; // registers global EventSource for @arkade-os/sdk; see src/polyfills.ts import { VersioningType } from '@nestjs/common'; import { NestFactory } from '@nestjs/core'; diff --git a/src/runtime-metrics.ts b/src/runtime-metrics.ts new file mode 100644 index 0000000000..189a4137c0 --- /dev/null +++ b/src/runtime-metrics.ts @@ -0,0 +1,121 @@ +import { metrics } from '@opentelemetry/api'; +import { EventLoopUtilization, IntervalHistogram, monitorEventLoopDelay, performance } from 'perf_hooks'; + +// Node runtime saturation metrics for dfx-api. +// +// Traces answer "how long did this request take"; they cannot answer "why". A request that +// waits behind a saturated event loop looks identical to one waiting on a slow query. These +// metrics close that gap: they measure whether the single JS thread had capacity at all. +// +// MonitorEventLoopService logs the same figures for humans, but it runs as a cron job — an +// HTTP-only instance (CRON_JOBS_ENABLED=false) registers no jobs and would therefore report +// nothing exactly where saturation matters most. The collection here is driven by the OTel +// metric reader instead, so it is independent of the scheduler and runs on every instance. +// +// Export travels the existing OTLP pipeline (see src/tracing.ts). With +// OTEL_EXPORTER_OTLP_ENDPOINT unset, no meter is registered and the app boots unchanged. + +const NS_PER_S = 1e9; + +/** Instrument names follow the OpenTelemetry Node.js runtime semantic conventions. */ +export const METER_NAME = 'dfx-api.runtime'; + +export interface EventLoopDelay { + min: number; + max: number; + mean: number; + p50: number; + p90: number; + p99: number; +} + +export interface EventLoopSample { + /** Fraction of the interval the loop was busy, 0..1. */ + utilization: number; + /** Delay percentiles in seconds, per semantic conventions. */ + delay: EventLoopDelay; +} + +/** + * Converts a histogram reading plus an utilization delta into one sample. + * + * Pure on purpose: the caller owns both the histogram reset and the utilization reference, so + * this stays unit-testable without timers. An empty histogram (no sample taken yet) reports + * zeros rather than the `Infinity`/`0` pair Node returns for `min`/`max` in that state, which + * would otherwise poison the exported series. + */ +export function toEventLoopSample(histogram: IntervalHistogram, utilization: number): EventLoopSample { + const empty = histogram.count === 0; + const seconds = (ns: number) => (empty ? 0 : ns / NS_PER_S); + + return { + utilization, + delay: { + min: seconds(histogram.min), + max: seconds(histogram.max), + mean: seconds(histogram.mean), + p50: seconds(histogram.percentile(50)), + p90: seconds(histogram.percentile(90)), + p99: seconds(histogram.percentile(99)), + }, + }; +} + +let started = false; + +/** + * Registers the runtime gauges. Returns false when tracing (and thus the meter provider) is + * disabled, so callers can tell "off by configuration" from "failed". + */ +export function startRuntimeMetrics(): boolean { + if (!process.env.OTEL_EXPORTER_OTLP_ENDPOINT) return false; + if (started) return true; + + const histogram = monitorEventLoopDelay({ resolution: 20 }); + histogram.enable(); + + let previousElu: EventLoopUtilization = performance.eventLoopUtilization(); + + const meter = metrics.getMeter(METER_NAME); + + const utilization = meter.createObservableGauge('nodejs.eventloop.utilization', { + description: 'Event loop utilization over the last export interval', + }); + const delayInstruments = { + min: meter.createObservableGauge('nodejs.eventloop.delay.min', { unit: 's' }), + max: meter.createObservableGauge('nodejs.eventloop.delay.max', { unit: 's' }), + mean: meter.createObservableGauge('nodejs.eventloop.delay.mean', { unit: 's' }), + p50: meter.createObservableGauge('nodejs.eventloop.delay.p50', { unit: 's' }), + p90: meter.createObservableGauge('nodejs.eventloop.delay.p90', { unit: 's' }), + p99: meter.createObservableGauge('nodejs.eventloop.delay.p99', { unit: 's' }), + }; + + // A batch callback fires once per collection for all instruments together. Registering one + // callback per gauge would reset the histogram six times per interval, so every gauge but + // the first would report an almost empty window. + meter.addBatchObservableCallback( + (observer) => { + const currentElu = performance.eventLoopUtilization(); + const intervalElu = performance.eventLoopUtilization(currentElu, previousElu); + const sample = toEventLoopSample(histogram, intervalElu.utilization); + + observer.observe(utilization, sample.utilization); + for (const [key, instrument] of Object.entries(delayInstruments)) { + observer.observe(instrument, sample.delay[key as keyof EventLoopDelay]); + } + + // Advance both windows together so delay and utilization always describe the same + // interval, and each export reports the interval just passed rather than the whole + // process lifetime. + previousElu = currentElu; + histogram.reset(); + }, + [utilization, ...Object.values(delayInstruments)], + ); + + started = true; + + return true; +} + +startRuntimeMetrics(); diff --git a/src/tracing.ts b/src/tracing.ts index 937f4f5fb1..dc8c3f7b62 100644 --- a/src/tracing.ts +++ b/src/tracing.ts @@ -1,6 +1,8 @@ import { SpanKind, SpanStatusCode } from '@opentelemetry/api'; import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node'; +import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'; import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; +import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; import { NodeSDK } from '@opentelemetry/sdk-node'; import { BatchSpanProcessor, ReadableSpan, SpanProcessor } from '@opentelemetry/sdk-trace-base'; @@ -61,6 +63,12 @@ export class ClientErrorSpanProcessor implements SpanProcessor { } } +/** + * Metric export interval. Matches the existing event-loop log cadence, which is short enough to + * catch multi-second stalls but long enough to keep series volume modest. + */ +export const METRIC_EXPORT_INTERVAL_MS = 10_000; + let sdk: NodeSDK | undefined; export function startTracing(): NodeSDK | undefined { @@ -74,6 +82,13 @@ export function startTracing(): NodeSDK | undefined { // processor so corrected statuses are what gets exported. The exporter // reads OTEL_EXPORTER_OTLP_ENDPOINT from the environment. spanProcessors: [new ClientErrorSpanProcessor(), new BatchSpanProcessor(new OTLPTraceExporter())], + // Metrics travel the same OTLP route as spans, so runtime saturation (see + // src/runtime-metrics.ts) needs no extra endpoint or scrape target. Spans measure how long + // work waited; these measure whether the process had CPU to run it at all. + metricReader: new PeriodicExportingMetricReader({ + exporter: new OTLPMetricExporter(), + exportIntervalMillis: METRIC_EXPORT_INTERVAL_MS, + }), instrumentations: [ getNodeAutoInstrumentations({ // Filesystem spans are pure noise for an API service. From 639ac6cd3fdc5bb8af91ba83fdad375f11d3bb77 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:08 +0200 Subject: [PATCH 03/86] Apply Prettier formatting to the runtime metrics test --- src/__tests__/runtime-metrics.spec.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/__tests__/runtime-metrics.spec.ts b/src/__tests__/runtime-metrics.spec.ts index 74e6c52cf1..583a629fdc 100644 --- a/src/__tests__/runtime-metrics.spec.ts +++ b/src/__tests__/runtime-metrics.spec.ts @@ -3,7 +3,9 @@ import { toEventLoopSample } from '../runtime-metrics'; const NS_PER_S = 1e9; -function fakeHistogram(values: Partial> & { count: number }): IntervalHistogram { +function fakeHistogram( + values: Partial> & { count: number }, +): IntervalHistogram { return { ...values, percentile: (p: number) => { From daeeb71e6a1bc995bd7257c5f1b3287466a1c5b7 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:10 +0200 Subject: [PATCH 04/86] Reject an empty CRON_JOBS_ENABLED instead of treating it as unset Only a variable that is absent entirely now means "run jobs". An empty value takes the dangerous path the surrounding comment claims to rule out: an `CRON_JOBS_ENABLED=` line in an env file or an unresolved `${VAR}` in a compose file would have re-registered the scheduler on an instance meant to serve HTTP only, and jobs without a declared `process` would then run on two instances at once. That accident is more likely than the misspelling the check was written for, so it fails the boot too. Also documents the variable in .env.example with an explicit value rather than the usual empty placeholder, which would now be rejected, and covers the env -> Config wiring instead of the parser alone: the cron service reads Config.cronJobsEnabled, and nothing verified that path end to end. The service test builds its configuration through ConfigService/GetConfig for the same reason - spreading GetConfig() dropped the prototype getters and left the global singleton structurally different from a real Configuration. Shares the telemetry-enabled check between tracing and runtime metrics so the two halves cannot drift apart, and aligns the sdk-metrics range with the baseline the rest of the OTel stack pins. --- .env.example | 5 ++ package-lock.json | 2 +- package.json | 2 +- .../cron-jobs-enabled.config.spec.ts | 55 ++++++++++++++++--- src/config/config.ts | 16 ++++-- src/runtime-metrics.ts | 7 ++- .../__tests__/dfx-cron.service.spec.ts | 18 ++++-- src/tracing.ts | 11 +++- 8 files changed, 92 insertions(+), 24 deletions(-) diff --git a/.env.example b/.env.example index 68300ecaac..bce6b8b0a5 100644 --- a/.env.example +++ b/.env.example @@ -343,3 +343,8 @@ REALUNIT_W2W_GAS_LOW_BALANCE_THRESHOLD=0.05 REQUEST_KNOWN_IPS= CRON_JOB_DELAY= + +# 'true' or 'false' only — an empty or misspelled value fails the boot on purpose, so that an +# instance meant to serve HTTP only never re-registers the scheduler. Omit the line entirely to +# run cron jobs, which is the default behaviour. +CRON_JOBS_ENABLED=true diff --git a/package-lock.json b/package-lock.json index 7f440e6dc2..6fc98b7c19 100644 --- a/package-lock.json +++ b/package-lock.json @@ -46,7 +46,7 @@ "@opentelemetry/auto-instrumentations-node": "^0.76.0", "@opentelemetry/exporter-metrics-otlp-http": "^0.218.0", "@opentelemetry/exporter-trace-otlp-http": "^0.218.0", - "@opentelemetry/sdk-metrics": "^2.10.0", + "@opentelemetry/sdk-metrics": "^2.7.1", "@opentelemetry/sdk-node": "^0.218.0", "@opentelemetry/sdk-trace-base": "^2.7.1", "@railgun-community/engine": "^9.4.0", diff --git a/package.json b/package.json index 799d020cf0..66bf716c81 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,7 @@ "@opentelemetry/auto-instrumentations-node": "^0.76.0", "@opentelemetry/exporter-metrics-otlp-http": "^0.218.0", "@opentelemetry/exporter-trace-otlp-http": "^0.218.0", - "@opentelemetry/sdk-metrics": "^2.10.0", + "@opentelemetry/sdk-metrics": "^2.7.1", "@opentelemetry/sdk-node": "^0.218.0", "@opentelemetry/sdk-trace-base": "^2.7.1", "@railgun-community/engine": "^9.4.0", diff --git a/src/config/__tests__/cron-jobs-enabled.config.spec.ts b/src/config/__tests__/cron-jobs-enabled.config.spec.ts index 24350a959e..c6b9c2ee77 100644 --- a/src/config/__tests__/cron-jobs-enabled.config.spec.ts +++ b/src/config/__tests__/cron-jobs-enabled.config.spec.ts @@ -1,9 +1,8 @@ -import { parseCronJobsEnabled } from '../config'; +import { Config, ConfigService, GetConfig, parseCronJobsEnabled } from '../config'; describe('parseCronJobsEnabled', () => { - it('defaults to enabled when unset, so existing environments keep running jobs', () => { + it('defaults to enabled when the variable is absent, so existing environments keep running jobs', () => { expect(parseCronJobsEnabled(undefined)).toBe(true); - expect(parseCronJobsEnabled('')).toBe(true); }); it('accepts the two valid values', () => { @@ -11,14 +10,54 @@ describe('parseCronJobsEnabled', () => { expect(parseCronJobsEnabled('false')).toBe(false); }); - it.each(['fals', 'False', 'FALSE', '0', 'no', 'off', 'disabled', ' false'])( + it.each(['', ' ', 'fals', 'False', 'FALSE', '0', 'no', 'off', 'disabled', ' false'])( 'throws on %p instead of silently enabling the scheduler', (value) => { - // The dangerous direction is a typo being read as "enabled": on an instance meant to be - // HTTP-only that would re-register every job, and jobs without a `process` (trades, - // referral credits, volume resets) would then run on two instances at once. Cron locks - // are per-process, so nothing else would catch it. Failing the boot is the safe outcome. + // The dangerous direction is a value being read as "enabled": on an instance meant to be + // HTTP-only that re-registers every job, and jobs without a `process` (trades, referral + // credits, volume resets) would then run on two instances at once. Cron locks are + // per-process, so nothing else would catch it. Failing the boot is the safe outcome. + // The empty string is included deliberately — an `CRON_JOBS_ENABLED=` line or an + // unresolved `${VAR}` is the likeliest accident of all. expect(() => parseCronJobsEnabled(value)).toThrow(/expected 'true' or 'false'/); }, ); }); + +describe('Config.cronJobsEnabled', () => { + const original = process.env.CRON_JOBS_ENABLED; + + afterEach(() => { + if (original == null) delete process.env.CRON_JOBS_ENABLED; + else process.env.CRON_JOBS_ENABLED = original; + + new ConfigService(GetConfig()); + }); + + // Covers the wiring env -> parseCronJobsEnabled -> Config that DfxCronService reads, + // which unit-testing the parser alone would leave unverified. + it.each([ + ['false', false], + ['true', true], + ])('maps CRON_JOBS_ENABLED=%s to %s', (value, expected) => { + process.env.CRON_JOBS_ENABLED = value; + + new ConfigService(GetConfig()); + + expect(Config.cronJobsEnabled).toBe(expected); + }); + + it('runs jobs when the variable is absent', () => { + delete process.env.CRON_JOBS_ENABLED; + + new ConfigService(GetConfig()); + + expect(Config.cronJobsEnabled).toBe(true); + }); + + it('refuses to build a configuration from an invalid value', () => { + process.env.CRON_JOBS_ENABLED = 'fals'; + + expect(() => GetConfig()).toThrow(/expected 'true' or 'false'/); + }); +}); diff --git a/src/config/config.ts b/src/config/config.ts index e677685e35..b79bb97c5b 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -1515,13 +1515,19 @@ export class Configuration { /** * Reads the CRON_JOBS_ENABLED flag. * - * Unset means "run jobs", which keeps every existing environment working unchanged. Any other - * value than 'true'/'false' throws instead of being coerced: a typo such as 'fals' would - * otherwise silently enable the scheduler on an instance meant to be HTTP-only, and duplicate - * job execution is far more damaging than a failed boot. + * Only an entirely unset variable means "run jobs", which keeps every existing environment + * working unchanged. Every other value must be exactly 'true' or 'false' and throws otherwise, + * rather than being coerced: on an instance meant to be HTTP-only, anything that silently reads + * as "enabled" re-registers the scheduler, and jobs without a declared `process` would then run + * on two instances at once. Duplicate execution of financial jobs is far more damaging than a + * failed boot. + * + * The empty string throws for that reason too, even though it is the more common accident — an + * `CRON_JOBS_ENABLED=` line in an env file, or an unresolved `${VAR}` in a compose file. Neither + * should quietly turn a job container back on. */ export function parseCronJobsEnabled(value?: string): boolean { - if (value == null || value === '') return true; + if (value == null) return true; if (value === 'true') return true; if (value === 'false') return false; diff --git a/src/runtime-metrics.ts b/src/runtime-metrics.ts index 189a4137c0..b426476869 100644 --- a/src/runtime-metrics.ts +++ b/src/runtime-metrics.ts @@ -1,5 +1,6 @@ import { metrics } from '@opentelemetry/api'; import { EventLoopUtilization, IntervalHistogram, monitorEventLoopDelay, performance } from 'perf_hooks'; +import { isTelemetryEnabled } from './tracing'; // Node runtime saturation metrics for dfx-api. // @@ -64,11 +65,11 @@ export function toEventLoopSample(histogram: IntervalHistogram, utilization: num let started = false; /** - * Registers the runtime gauges. Returns false when tracing (and thus the meter provider) is - * disabled, so callers can tell "off by configuration" from "failed". + * Registers the runtime gauges. Returns whether they are active: false means telemetry is + * switched off by configuration, in which case there is no meter provider to register with. */ export function startRuntimeMetrics(): boolean { - if (!process.env.OTEL_EXPORTER_OTLP_ENDPOINT) return false; + if (!isTelemetryEnabled()) return false; if (started) return true; const histogram = monitorEventLoopDelay({ resolution: 20 }); diff --git a/src/shared/services/__tests__/dfx-cron.service.spec.ts b/src/shared/services/__tests__/dfx-cron.service.spec.ts index 9f7fe92877..fea599f9cb 100644 --- a/src/shared/services/__tests__/dfx-cron.service.spec.ts +++ b/src/shared/services/__tests__/dfx-cron.service.spec.ts @@ -5,12 +5,12 @@ jest.mock('cron', () => ({ })); import { createMock } from '@golevelup/ts-jest'; -import { CronExpression, SchedulerRegistry } from '@nestjs/schedule'; import { DiscoveryService, MetadataScanner } from '@nestjs/core'; -import { Config, ConfigService, GetConfig } from 'src/config/config'; +import { CronExpression, SchedulerRegistry } from '@nestjs/schedule'; +import { ConfigService, GetConfig } from 'src/config/config'; import { DFX_CRONJOB_PARAMS, DfxCronParams } from 'src/shared/utils/cron'; -import { Process } from '../process.service'; import { DfxCronService } from '../dfx-cron.service'; +import { Process } from '../process.service'; /** Builds a provider instance carrying @DfxCron metadata, as the decorator would. */ function providerWithJob(methodName: string, params: DfxCronParams): { instance: object } { @@ -44,6 +44,8 @@ function buildService(providers: { instance: object }[]): { } describe('DfxCronService', () => { + const original = process.env.CRON_JOBS_ENABLED; + const configuredJobs = [ providerWithJob('withProcess', { expression: CronExpression.EVERY_MINUTE, process: Process.MONITOR_EVENT_LOOP }), // A job without `process` — DISABLED_PROCESSES cannot stop this one, only the global switch can. @@ -52,11 +54,16 @@ describe('DfxCronService', () => { afterEach(() => { jest.clearAllMocks(); + + if (original == null) delete process.env.CRON_JOBS_ENABLED; + else process.env.CRON_JOBS_ENABLED = original; + new ConfigService(GetConfig()); }); it('registers jobs when cron is enabled', () => { - new ConfigService({ ...GetConfig(), cronJobsEnabled: true } as typeof Config); + process.env.CRON_JOBS_ENABLED = 'true'; + new ConfigService(GetConfig()); const { service, scheduler } = buildService(configuredJobs); service.onModuleInit(); @@ -69,7 +76,8 @@ describe('DfxCronService', () => { // The safety property of the HTTP-only instance: were a job without `process` still // registered here, it would run on both the HTTP and the job instance simultaneously. // Cron locks are per-process, so duplicate execution would go unnoticed. - new ConfigService({ ...GetConfig(), cronJobsEnabled: false } as typeof Config); + process.env.CRON_JOBS_ENABLED = 'false'; + new ConfigService(GetConfig()); const { service, scheduler } = buildService(configuredJobs); service.onModuleInit(); diff --git a/src/tracing.ts b/src/tracing.ts index dc8c3f7b62..4f8f0da47d 100644 --- a/src/tracing.ts +++ b/src/tracing.ts @@ -71,9 +71,18 @@ export const METRIC_EXPORT_INTERVAL_MS = 10_000; let sdk: NodeSDK | undefined; +/** + * Whether telemetry export is configured at all. Shared with src/runtime-metrics.ts so both + * halves switch on the same condition — the metrics live in the meter provider this module + * registers, and a second copy of the check would drift the moment this one changes. + */ +export function isTelemetryEnabled(): boolean { + return Boolean(process.env.OTEL_EXPORTER_OTLP_ENDPOINT); +} + export function startTracing(): NodeSDK | undefined { // Disabled unless a collector endpoint is configured (e.g. on LOC / in tests). - if (!process.env.OTEL_EXPORTER_OTLP_ENDPOINT) return undefined; + if (!isTelemetryEnabled()) return undefined; if (sdk) return sdk; sdk = new NodeSDK({ From 8c473133cbb21483e2221db175a61e9f3a0bca85 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:12 +0200 Subject: [PATCH 05/86] Keep per-instance housekeeping running when cron is disabled The switch was too coarse. Turning off every job on the HTTP instance also turned off the jobs that must run precisely there, because they refresh process-local state that requests on that same instance read: - resyncDeniedJwtAddresses / resyncDeniedJwtAccounts rebuild the in-memory JWT denylists every 30 s. Without them the lists stay frozen at boot, so blocking an account would no longer revoke its live tokens on the instance that serves HTTP until the next restart. The lookup fails open on an empty set, so nothing would have surfaced the gap. - resyncDisabledProcesses applies the database-driven kill switch and safety mode, which gate several HTTP endpoints. - updateCache holds the transaction specifications behind quote and limit calculation; checkLists and processCleanupMailSecretCache expire local caches that only grow where the traffic arrives. DfxCron therefore takes a `perInstance` flag for work whose effect is confined to its own process, and the switch now skips only the rest. Running such a job on every instance is harmless by construction; anything that writes to the database or drives business forward stays global and is skipped as before. Also moves the two remaining native @Cron decorators to @DfxCron. They were registered by the Nest scheduler directly and bypassed the switch entirely, which made the claim that an HTTP-only instance registers nothing false. Their process guards move into the decorator, where DfxCronService evaluates them anyway, so the bodies lose their early return. --- .../controllers/exchange.controller.ts | 2 +- .../__tests__/dfx-cron.service.spec.ts | 34 +++++++++++++++---- src/shared/services/dfx-cron.service.ts | 27 ++++++++++----- src/shared/services/process.service.ts | 6 ++-- src/shared/utils/cron.ts | 13 +++++++ .../controllers/transaction.controller.ts | 2 +- .../generic/user/models/auth/auth.service.ts | 2 +- .../models/user-data/user-data.service.ts | 2 +- .../payment/services/transaction-helper.ts | 2 +- .../services/transaction-request.service.ts | 16 +++------ src/tracing.ts | 29 +++++++++++++--- 11 files changed, 98 insertions(+), 37 deletions(-) diff --git a/src/integration/exchange/controllers/exchange.controller.ts b/src/integration/exchange/controllers/exchange.controller.ts index c78d5bb5b5..e01f856443 100644 --- a/src/integration/exchange/controllers/exchange.controller.ts +++ b/src/integration/exchange/controllers/exchange.controller.ts @@ -171,7 +171,7 @@ export class ExchangeController { } // --- JOBS --- // - @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800 }) + @DfxCron(CronExpression.EVERY_30_SECONDS, { perInstance: true, timeout: 1800 }) async checkTrades() { const openTrades = Object.values(this.trades).filter(({ status }) => status === TradeStatus.OPEN); for (const trade of openTrades) { diff --git a/src/shared/services/__tests__/dfx-cron.service.spec.ts b/src/shared/services/__tests__/dfx-cron.service.spec.ts index fea599f9cb..c7dab9131d 100644 --- a/src/shared/services/__tests__/dfx-cron.service.spec.ts +++ b/src/shared/services/__tests__/dfx-cron.service.spec.ts @@ -50,8 +50,15 @@ describe('DfxCronService', () => { providerWithJob('withProcess', { expression: CronExpression.EVERY_MINUTE, process: Process.MONITOR_EVENT_LOOP }), // A job without `process` — DISABLED_PROCESSES cannot stop this one, only the global switch can. providerWithJob('withoutProcess', { expression: CronExpression.EVERY_MINUTE }), + // Per-instance housekeeping: must survive the switch, because it refreshes state that + // requests on this very instance read. + providerWithJob('perInstanceJob', { expression: CronExpression.EVERY_MINUTE, perInstance: true }), ]; + function registeredJobNames(scheduler: SchedulerRegistry): string[] { + return (scheduler.addCronJob as jest.Mock).mock.calls.map(([name]) => name as string); + } + afterEach(() => { jest.clearAllMocks(); @@ -61,18 +68,18 @@ describe('DfxCronService', () => { new ConfigService(GetConfig()); }); - it('registers jobs when cron is enabled', () => { + it('registers every job when cron is enabled', () => { process.env.CRON_JOBS_ENABLED = 'true'; new ConfigService(GetConfig()); const { service, scheduler } = buildService(configuredJobs); service.onModuleInit(); - expect(scheduler.addCronJob).toHaveBeenCalledTimes(2); - expect(mockStart).toHaveBeenCalledTimes(2); + expect(scheduler.addCronJob).toHaveBeenCalledTimes(3); + expect(mockStart).toHaveBeenCalledTimes(3); }); - it('registers no job at all when cron is disabled, including jobs without a process', () => { + it('drops global jobs when cron is disabled, including those without a process', () => { // The safety property of the HTTP-only instance: were a job without `process` still // registered here, it would run on both the HTTP and the job instance simultaneously. // Cron locks are per-process, so duplicate execution would go unnoticed. @@ -82,7 +89,22 @@ describe('DfxCronService', () => { const { service, scheduler } = buildService(configuredJobs); service.onModuleInit(); - expect(scheduler.addCronJob).not.toHaveBeenCalled(); - expect(mockStart).not.toHaveBeenCalled(); + expect(registeredJobNames(scheduler)).not.toContain('Object::withProcess'); + expect(registeredJobNames(scheduler)).not.toContain('Object::withoutProcess'); + }); + + it('keeps per-instance housekeeping when cron is disabled', () => { + // The counterpart safety property: jobs marked perInstance refresh process-local state + // (JWT denylists, the disabled-process map, local caches) that requests on THIS instance + // read. Dropping them here would freeze that state at boot — a revoked token would keep + // working on the HTTP instance until the next restart. + process.env.CRON_JOBS_ENABLED = 'false'; + new ConfigService(GetConfig()); + + const { service, scheduler } = buildService(configuredJobs); + service.onModuleInit(); + + expect(registeredJobNames(scheduler)).toEqual(['Object::perInstanceJob']); + expect(mockStart).toHaveBeenCalledTimes(1); }); }); diff --git a/src/shared/services/dfx-cron.service.ts b/src/shared/services/dfx-cron.service.ts index 55f62ffe74..fd717335e9 100644 --- a/src/shared/services/dfx-cron.service.ts +++ b/src/shared/services/dfx-cron.service.ts @@ -28,13 +28,7 @@ export class DfxCronService implements OnModuleInit { ) {} onModuleInit() { - // HTTP-only instances register nothing at all. Returning here (rather than skipping jobs - // individually) also covers jobs that declare no `process` and would otherwise stay active - // despite DISABLED_PROCESSES — on a second instance they would run twice. - if (!Config.cronJobsEnabled) { - this.logger.info('Cron jobs disabled on this instance (CRON_JOBS_ENABLED=false), registering none'); - return; - } + let skipped = 0; this.discovery .getProviders() @@ -54,8 +48,25 @@ export class DfxCronService implements OnModuleInit { }; }) .filter((data) => data.params) - .forEach((data) => this.addCronJob(data)); + .forEach((data) => { + // On an HTTP-only instance the global jobs belong to the job instance, but + // per-instance housekeeping must still run here: it refreshes process-local state + // such as the JWT denylists and the disabled-process map, which HTTP requests read + // on this very instance. Skipping those would freeze them at boot. + if (!Config.cronJobsEnabled && !data.params.perInstance) { + skipped++; + return; + } + + this.addCronJob(data); + }); }); + + if (skipped) { + this.logger.info( + `Cron jobs disabled on this instance (CRON_JOBS_ENABLED=false), skipped ${skipped} global job(s)`, + ); + } } private addCronJob(data: CronJobData) { diff --git a/src/shared/services/process.service.ts b/src/shared/services/process.service.ts index 4a9f1ba2e5..44defcb4ac 100644 --- a/src/shared/services/process.service.ts +++ b/src/shared/services/process.service.ts @@ -180,7 +180,7 @@ export class ProcessService implements OnModuleInit { await this.resyncStaffKycClearance(); } - @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800 }) + @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800, perInstance: true }) async resyncDisabledProcesses(): Promise { const allDisabledProcesses = [ ...(await this.settingService.getDisabledProcesses()), @@ -190,13 +190,13 @@ export class ProcessService implements OnModuleInit { DisabledProcesses = this.listToMap(allDisabledProcesses); } - @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800 }) + @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800, perInstance: true }) async resyncDeniedJwtAddresses(): Promise { const list = await this.settingService.getDeniedJwtAddresses(); DeniedJwtAddresses = new Set(list.map((a) => a.toLowerCase())); } - @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800 }) + @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800, perInstance: true }) async resyncDeniedJwtAccounts(): Promise { const list = await this.settingService.getDeniedJwtAccounts(); DeniedJwtAccounts = new Set(list); diff --git a/src/shared/utils/cron.ts b/src/shared/utils/cron.ts index 7faa755747..e5afb6565a 100644 --- a/src/shared/utils/cron.ts +++ b/src/shared/utils/cron.ts @@ -6,6 +6,19 @@ export interface DfxCronOptParams { process?: Process; useDelay?: boolean; timeout?: number; + /** + * Marks a job as per-instance housekeeping that must run in every process, even where the + * scheduler is otherwise switched off (CRON_JOBS_ENABLED=false). + * + * Use it only for work whose effect is confined to the current process: refreshing an + * in-memory copy of global state, or expiring a local cache. Running it twice must be + * harmless by construction, because it will run on every instance. + * + * Anything that writes to the database or drives business forward is NOT per-instance — such + * a job would then execute once per instance, and cron locks are per-process and cannot + * prevent that. + */ + perInstance?: boolean; } export type DfxCronExpression = CronExpression | CustomCronExpression; diff --git a/src/subdomains/core/history/controllers/transaction.controller.ts b/src/subdomains/core/history/controllers/transaction.controller.ts index e1135c54fd..f158b7b9a4 100644 --- a/src/subdomains/core/history/controllers/transaction.controller.ts +++ b/src/subdomains/core/history/controllers/transaction.controller.ts @@ -116,7 +116,7 @@ export class TransactionController { ) {} // --- JOBS --- // - @DfxCron(CronExpression.EVERY_MINUTE) + @DfxCron(CronExpression.EVERY_MINUTE, { perInstance: true }) checkLists() { for (const [key, refundData] of this.refundList.entries()) { if (!this.isRefundDataValid(refundData)) this.refundList.delete(key); diff --git a/src/subdomains/generic/user/models/auth/auth.service.ts b/src/subdomains/generic/user/models/auth/auth.service.ts index 258aa075b0..ab5e664440 100644 --- a/src/subdomains/generic/user/models/auth/auth.service.ts +++ b/src/subdomains/generic/user/models/auth/auth.service.ts @@ -98,7 +98,7 @@ export class AuthService { @Inject(forwardRef(() => KycService)) private readonly kycService: KycService, ) {} - @DfxCron(CronExpression.EVERY_MINUTE) + @DfxCron(CronExpression.EVERY_MINUTE, { perInstance: true }) checkLists() { for (const [key, challenge] of this.challengeList.entries()) { if (!this.isChallengeValid(challenge)) { diff --git a/src/subdomains/generic/user/models/user-data/user-data.service.ts b/src/subdomains/generic/user/models/user-data/user-data.service.ts index 0729bfa974..f619220797 100644 --- a/src/subdomains/generic/user/models/user-data/user-data.service.ts +++ b/src/subdomains/generic/user/models/user-data/user-data.service.ts @@ -833,7 +833,7 @@ export class UserDataService { return this.doUpdateUserMail(userData, cacheEntry.mail); } - @DfxCron(CronExpression.EVERY_MINUTE) + @DfxCron(CronExpression.EVERY_MINUTE, { perInstance: true }) processCleanupMailSecretCache(): void { const now = new Date(); diff --git a/src/subdomains/supporting/payment/services/transaction-helper.ts b/src/subdomains/supporting/payment/services/transaction-helper.ts index b848730a9d..2411db46bb 100644 --- a/src/subdomains/supporting/payment/services/transaction-helper.ts +++ b/src/subdomains/supporting/payment/services/transaction-helper.ts @@ -87,7 +87,7 @@ export class TransactionHelper implements OnModuleInit { void this.updateCache(); } - @DfxCron(CronExpression.EVERY_5_MINUTES) + @DfxCron(CronExpression.EVERY_5_MINUTES, { perInstance: true }) async updateCache() { this.transactionSpecifications = await this.specRepo.find(); } diff --git a/src/subdomains/supporting/payment/services/transaction-request.service.ts b/src/subdomains/supporting/payment/services/transaction-request.service.ts index a5505a760c..41cc114058 100644 --- a/src/subdomains/supporting/payment/services/transaction-request.service.ts +++ b/src/subdomains/supporting/payment/services/transaction-request.service.ts @@ -1,13 +1,13 @@ import { ForbiddenException, Inject, Injectable, NotFoundException, forwardRef } from '@nestjs/common'; -import { Cron, CronExpression } from '@nestjs/schedule'; +import { CronExpression } from '@nestjs/schedule'; import { Config } from 'src/config/config'; import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.enum'; import { SiftService } from 'src/integration/sift/services/sift.service'; import { AssetService } from 'src/shared/models/asset/asset.service'; import { FiatService } from 'src/shared/models/fiat/fiat.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; -import { DisabledProcess, Process } from 'src/shared/services/process.service'; -import { Lock } from 'src/shared/utils/lock'; +import { Process } from 'src/shared/services/process.service'; +import { DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { BuyService } from 'src/subdomains/core/buy-crypto/routes/buy/buy.service'; import { BuyPaymentInfoDto } from 'src/subdomains/core/buy-crypto/routes/buy/dto/buy-payment-info.dto'; @@ -49,20 +49,14 @@ export class TransactionRequestService { private readonly swapService: SwapService, ) {} - @Cron(CronExpression.EVERY_MINUTE) - @Lock(7200) + @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.TX_REQUEST, timeout: 7200 }) async txRequestStatusSync() { - if (DisabledProcess(Process.TX_REQUEST)) return; - await this.syncStatus(); await this.deleteOldTxRequests(); } - @Cron(CronExpression.EVERY_DAY_AT_3AM) - @Lock(7200) + @DfxCron(CronExpression.EVERY_DAY_AT_3AM, { process: Process.TX_REQUEST_WAITING_EXPIRY, timeout: 7200 }) async txRequestWaitingExpiryCheck() { - if (DisabledProcess(Process.TX_REQUEST_WAITING_EXPIRY)) return; - const expiryDate = Util.daysBefore(Config.txRequestWaitingExpiryDays); const entities = await this.transactionRequestRepo.findBy({ status: TransactionRequestStatus.WAITING_FOR_PAYMENT, diff --git a/src/tracing.ts b/src/tracing.ts index 4f8f0da47d..6cb61880b7 100644 --- a/src/tracing.ts +++ b/src/tracing.ts @@ -64,10 +64,25 @@ export class ClientErrorSpanProcessor implements SpanProcessor { } /** - * Metric export interval. Matches the existing event-loop log cadence, which is short enough to - * catch multi-second stalls but long enough to keep series volume modest. + * Metric export interval in milliseconds. + * + * Deliberately left to OTEL_METRIC_EXPORT_INTERVAL (the SDK's own variable, default 60s) rather + * than pinned in code. An explicit reader takes precedence over the SDK's env handling, so a + * hardcoded value would silently disable that knob — and a shorter interval costs a full + * collect-and-export of *every* instrument, including the auto-instrumentation histograms, on + * the very event loop this is meant to keep free. */ -export const METRIC_EXPORT_INTERVAL_MS = 10_000; +export function metricExportIntervalMs(): number | undefined { + const raw = process.env.OTEL_METRIC_EXPORT_INTERVAL; + if (raw == null || raw === '') return undefined; + + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`Invalid OTEL_METRIC_EXPORT_INTERVAL value '${raw}': expected a positive number of milliseconds`); + } + + return parsed; +} let sdk: NodeSDK | undefined; @@ -85,6 +100,8 @@ export function startTracing(): NodeSDK | undefined { if (!isTelemetryEnabled()) return undefined; if (sdk) return sdk; + const intervalMs = metricExportIntervalMs(); + sdk = new NodeSDK({ serviceName: 'dfx-api', // The 4xx-not-a-failure processor runs before the exporting batch @@ -94,9 +111,13 @@ export function startTracing(): NodeSDK | undefined { // Metrics travel the same OTLP route as spans, so runtime saturation (see // src/runtime-metrics.ts) needs no extra endpoint or scrape target. Spans measure how long // work waited; these measure whether the process had CPU to run it at all. + // + // The reader is declared explicitly because the gauges need a meter provider that is + // guaranteed to exist; the interval stays env-driven so this does not quietly change the + // export cadence the SDK would otherwise use. metricReader: new PeriodicExportingMetricReader({ exporter: new OTLPMetricExporter(), - exportIntervalMillis: METRIC_EXPORT_INTERVAL_MS, + ...(intervalMs == null ? {} : { exportIntervalMillis: intervalMs }), }), instrumentations: [ getNodeAutoInstrumentations({ From 7aeced4948e623243f559c01647142d2ef98c5af Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:14 +0200 Subject: [PATCH 06/86] Mark the remaining process-local jobs as per-instance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first pass only reviewed jobs without a `process` parameter, which missed the same class of job among those that declare one. Found by scanning every @DfxCron body for writes to process-local state instead of reading the list again: - payment-link-fee.updateFees holds the fee cache behind LNURL-Pay and payment link quotes. Entries expire after 5 minutes, so on an HTTP-only instance every such request would have started failing five minutes after boot. - tfa.processCleanupSecretCache is the only thing enforcing expiry of mail 2FA codes and enrollment secrets — verify() never checks the expiry date itself. Without it those stay valid indefinitely on the instance serving the requests. - auth-lnurl.processCleanupAccessToken likewise enforces the 30-second window in which a minted JWT is retrievable via the publicly known k1; status() does not check the timestamp. - statistic.doUpdate, binance-pay.updateCertificates, monitorEventLoop and processCleanupAuthCache follow the same pattern. The scan also flagged five jobs that merely set a "warning already logged" flag while doing global work (pay-in registration, fiat sync, monitoring observers). Those deliberately stay global: marking them would have processed incoming funds twice. Known limitation, deliberately not addressed here: /health reads the observer state from memory, and the observers write to the database, so they cannot simply run everywhere. On an HTTP-only instance the endpoint therefore reports the snapshot loaded at boot. Making it re-read that snapshot touches observer semantics and belongs in its own change. --- .../services/binance-pay.service.ts | 2 +- .../monitoring/monitor-event-loop.service.ts | 2 +- .../services/payment-link-fee.service.ts | 230 ++++---- .../core/statistic/statistic.service.ts | 2 +- .../generic/kyc/services/tfa.service.ts | 548 +++++++++--------- .../user/models/auth/auth-lnurl.service.ts | 318 +++++----- 6 files changed, 551 insertions(+), 551 deletions(-) diff --git a/src/integration/binance-pay/services/binance-pay.service.ts b/src/integration/binance-pay/services/binance-pay.service.ts index 9de4b396e2..f7fa5371d4 100644 --- a/src/integration/binance-pay/services/binance-pay.service.ts +++ b/src/integration/binance-pay/services/binance-pay.service.ts @@ -225,7 +225,7 @@ export class BinancePayService implements C2BPaymentLinkProvider { try { const headers = this.getHeaders({}); diff --git a/src/subdomains/core/monitoring/monitor-event-loop.service.ts b/src/subdomains/core/monitoring/monitor-event-loop.service.ts index 3a531a13c2..efc5dc3ba3 100644 --- a/src/subdomains/core/monitoring/monitor-event-loop.service.ts +++ b/src/subdomains/core/monitoring/monitor-event-loop.service.ts @@ -22,7 +22,7 @@ export class MonitorEventLoopService implements OnModuleDestroy { this.histogram.disable(); } - @DfxCron(CronExpression.EVERY_10_SECONDS, { process: Process.MONITOR_EVENT_LOOP }) + @DfxCron(CronExpression.EVERY_10_SECONDS, { perInstance: true, process: Process.MONITOR_EVENT_LOOP }) monitorEventLoop(): void { const toMs = (ns: number) => Math.round(ns / 1e6); diff --git a/src/subdomains/core/payment-link/services/payment-link-fee.service.ts b/src/subdomains/core/payment-link/services/payment-link-fee.service.ts index ef2c786d66..8bcc8e2391 100644 --- a/src/subdomains/core/payment-link/services/payment-link-fee.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-fee.service.ts @@ -1,115 +1,115 @@ -import { Injectable, OnModuleInit } from '@nestjs/common'; -import { CronExpression } from '@nestjs/schedule'; -import { Environment, GetConfig } from 'src/config/config'; -import { MIN_FEE_RATE_SAT_VB } from 'src/integration/blockchain/bitcoin/services/bitcoin-based-fee.service'; -import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.enum'; -import { PaymentLinkBlockchains } from 'src/integration/blockchain/shared/util/blockchain.util'; -import { DfxLogger } from 'src/shared/services/dfx-logger'; -import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; -import { Util } from 'src/shared/utils/util'; -import { PayoutBitcoinService } from 'src/subdomains/supporting/payout/services/payout-bitcoin.service'; -import { PayoutFiroService } from 'src/subdomains/supporting/payout/services/payout-firo.service'; -import { BlockchainRegistryService } from '../../../../integration/blockchain/shared/services/blockchain-registry.service'; - -interface FeeCacheData { - timestamp: Date; - fee: number; -} - -@Injectable() -export class PaymentLinkFeeService implements OnModuleInit { - private readonly logger = new DfxLogger(PaymentLinkFeeService); - - private static readonly MINUTES_5 = 5 * 60; - - private readonly feeCache: Map; - - constructor( - private readonly blockchainRegistryService: BlockchainRegistryService, - private readonly payoutBitcoinService: PayoutBitcoinService, - private readonly payoutFiroService: PayoutFiroService, - ) { - this.feeCache = new Map(); - } - - onModuleInit() { - void this.updateFees(); - } - - // --- JOBS --- // - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.UPDATE_BLOCKCHAIN_FEE }) - async updateFees(): Promise { - if (GetConfig().environment === Environment.LOC) return; - - for (const blockchain of PaymentLinkBlockchains) { - try { - const fee = await this.calculateFee(blockchain); - this.feeCache.set(blockchain, { - timestamp: new Date(), - fee, - }); - } catch (e) { - this.feeCache.delete(blockchain); - this.logger.error(`Failed to get fee for blockchain ${blockchain}:`, e); - } - } - } - - private async calculateFee(blockchain: Blockchain): Promise { - switch (blockchain) { - case Blockchain.BINANCE_PAY: - case Blockchain.KUCOIN_PAY: - case Blockchain.LIGHTNING: - case Blockchain.MONERO: - case Blockchain.ZANO: - case Blockchain.SOLANA: - case Blockchain.TRON: - case Blockchain.CARDANO: - case Blockchain.INTERNET_COMPUTER: - return 0; - - case Blockchain.ETHEREUM: - case Blockchain.SEPOLIA: - case Blockchain.ARBITRUM: - case Blockchain.OPTIMISM: - case Blockchain.BASE: - case Blockchain.GNOSIS: - case Blockchain.POLYGON: - case Blockchain.BINANCE_SMART_CHAIN: { - const client = this.blockchainRegistryService.getEvmClient(blockchain); - return +(await client.getRecommendedGasPrice()); - } - - // The customer minimum is the network's own minimum for an inbound payment to confirm — it - // must NOT include the CPFP/default margin from getSendFeeRate, which exists only for DFX's - // own outbound spends. The value differs per chain because the chains do, but neither carries - // the payout margin. - case Blockchain.BITCOIN: - // Bitcoin fees are user-adjustable and the chain can congest, so use the recommended - // (next-block) rate, which adapts to congestion — floored at the relay minimum so the - // advertised minimum is always relayable. - return Math.max(await this.payoutBitcoinService.getRecommendedFeeRate(), MIN_FEE_RATE_SAT_VB); - - case Blockchain.FIRO: - // Same principle as Bitcoin: Firo's own next-block rate without the payout margin, floored - // at the relay minimum so it stays relayable. The current OCP deposit address is transparent, - // so a Stack Wallet payment is a Spark-spend to it, whose fee sits at the relay floor and - // cannot be raised; Firo does not congest and its node usually returns no estimate, so this - // resolves to the relay floor in practice — exactly what that Spark-spend pays. A dedicated - // relay-floor cap belongs here only once a Spark `sm1…` deposit address is deployed, whose - // protocol-capped fee cannot follow a congestion-adaptive minimum. - return Math.max(await this.payoutFiroService.getRecommendedFeeRate(), MIN_FEE_RATE_SAT_VB); - } - } - - // --- PUBLIC METHODS --- // - async getMinFee(blockchain: Blockchain): Promise { - const cacheData = this.feeCache.get(blockchain); - if (!cacheData) return; - - if (Util.secondsDiff(cacheData.timestamp) > PaymentLinkFeeService.MINUTES_5) return; - - return cacheData.fee; - } -} +import { Injectable, OnModuleInit } from '@nestjs/common'; +import { CronExpression } from '@nestjs/schedule'; +import { Environment, GetConfig } from 'src/config/config'; +import { MIN_FEE_RATE_SAT_VB } from 'src/integration/blockchain/bitcoin/services/bitcoin-based-fee.service'; +import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.enum'; +import { PaymentLinkBlockchains } from 'src/integration/blockchain/shared/util/blockchain.util'; +import { DfxLogger } from 'src/shared/services/dfx-logger'; +import { Process } from 'src/shared/services/process.service'; +import { DfxCron } from 'src/shared/utils/cron'; +import { Util } from 'src/shared/utils/util'; +import { PayoutBitcoinService } from 'src/subdomains/supporting/payout/services/payout-bitcoin.service'; +import { PayoutFiroService } from 'src/subdomains/supporting/payout/services/payout-firo.service'; +import { BlockchainRegistryService } from '../../../../integration/blockchain/shared/services/blockchain-registry.service'; + +interface FeeCacheData { + timestamp: Date; + fee: number; +} + +@Injectable() +export class PaymentLinkFeeService implements OnModuleInit { + private readonly logger = new DfxLogger(PaymentLinkFeeService); + + private static readonly MINUTES_5 = 5 * 60; + + private readonly feeCache: Map; + + constructor( + private readonly blockchainRegistryService: BlockchainRegistryService, + private readonly payoutBitcoinService: PayoutBitcoinService, + private readonly payoutFiroService: PayoutFiroService, + ) { + this.feeCache = new Map(); + } + + onModuleInit() { + void this.updateFees(); + } + + // --- JOBS --- // + @DfxCron(CronExpression.EVERY_MINUTE, { perInstance: true, process: Process.UPDATE_BLOCKCHAIN_FEE }) + async updateFees(): Promise { + if (GetConfig().environment === Environment.LOC) return; + + for (const blockchain of PaymentLinkBlockchains) { + try { + const fee = await this.calculateFee(blockchain); + this.feeCache.set(blockchain, { + timestamp: new Date(), + fee, + }); + } catch (e) { + this.feeCache.delete(blockchain); + this.logger.error(`Failed to get fee for blockchain ${blockchain}:`, e); + } + } + } + + private async calculateFee(blockchain: Blockchain): Promise { + switch (blockchain) { + case Blockchain.BINANCE_PAY: + case Blockchain.KUCOIN_PAY: + case Blockchain.LIGHTNING: + case Blockchain.MONERO: + case Blockchain.ZANO: + case Blockchain.SOLANA: + case Blockchain.TRON: + case Blockchain.CARDANO: + case Blockchain.INTERNET_COMPUTER: + return 0; + + case Blockchain.ETHEREUM: + case Blockchain.SEPOLIA: + case Blockchain.ARBITRUM: + case Blockchain.OPTIMISM: + case Blockchain.BASE: + case Blockchain.GNOSIS: + case Blockchain.POLYGON: + case Blockchain.BINANCE_SMART_CHAIN: { + const client = this.blockchainRegistryService.getEvmClient(blockchain); + return +(await client.getRecommendedGasPrice()); + } + + // The customer minimum is the network's own minimum for an inbound payment to confirm — it + // must NOT include the CPFP/default margin from getSendFeeRate, which exists only for DFX's + // own outbound spends. The value differs per chain because the chains do, but neither carries + // the payout margin. + case Blockchain.BITCOIN: + // Bitcoin fees are user-adjustable and the chain can congest, so use the recommended + // (next-block) rate, which adapts to congestion — floored at the relay minimum so the + // advertised minimum is always relayable. + return Math.max(await this.payoutBitcoinService.getRecommendedFeeRate(), MIN_FEE_RATE_SAT_VB); + + case Blockchain.FIRO: + // Same principle as Bitcoin: Firo's own next-block rate without the payout margin, floored + // at the relay minimum so it stays relayable. The current OCP deposit address is transparent, + // so a Stack Wallet payment is a Spark-spend to it, whose fee sits at the relay floor and + // cannot be raised; Firo does not congest and its node usually returns no estimate, so this + // resolves to the relay floor in practice — exactly what that Spark-spend pays. A dedicated + // relay-floor cap belongs here only once a Spark `sm1…` deposit address is deployed, whose + // protocol-capped fee cannot follow a congestion-adaptive minimum. + return Math.max(await this.payoutFiroService.getRecommendedFeeRate(), MIN_FEE_RATE_SAT_VB); + } + } + + // --- PUBLIC METHODS --- // + async getMinFee(blockchain: Blockchain): Promise { + const cacheData = this.feeCache.get(blockchain); + if (!cacheData) return; + + if (Util.secondsDiff(cacheData.timestamp) > PaymentLinkFeeService.MINUTES_5) return; + + return cacheData.fee; + } +} diff --git a/src/subdomains/core/statistic/statistic.service.ts b/src/subdomains/core/statistic/statistic.service.ts index 889efd607e..85b1231089 100644 --- a/src/subdomains/core/statistic/statistic.service.ts +++ b/src/subdomains/core/statistic/statistic.service.ts @@ -25,7 +25,7 @@ export class StatisticService implements OnModuleInit { void this.doUpdate(); } - @DfxCron(CronExpression.EVERY_HOUR, { process: Process.UPDATE_STATISTIC, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_HOUR, { perInstance: true, process: Process.UPDATE_STATISTIC, timeout: 7200 }) async doUpdate(): Promise { this.statistic = { totalVolume: { diff --git a/src/subdomains/generic/kyc/services/tfa.service.ts b/src/subdomains/generic/kyc/services/tfa.service.ts index 6870109209..e84636072c 100644 --- a/src/subdomains/generic/kyc/services/tfa.service.ts +++ b/src/subdomains/generic/kyc/services/tfa.service.ts @@ -1,274 +1,274 @@ -import { - ConflictException, - ForbiddenException, - Inject, - Injectable, - NotFoundException, - ServiceUnavailableException, - forwardRef, -} from '@nestjs/common'; -import { TfaRequiredException } from '../exceptions/tfa-required.exception'; -import { CronExpression } from '@nestjs/schedule'; -import { generateSecret, verifyToken } from 'node-2fa'; -import { DfxLogger } from 'src/shared/services/dfx-logger'; -import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; -import { Util } from 'src/shared/utils/util'; -import { TfaLogRepository } from 'src/subdomains/generic/kyc/repositories/tfa-log.repository'; -import { MailContext, MailType } from 'src/subdomains/supporting/notification/enums'; -import { MailKey, MailTranslationKey } from 'src/subdomains/supporting/notification/factories/mail.factory'; -import { NotificationService } from 'src/subdomains/supporting/notification/services/notification.service'; -import { MoreThan } from 'typeorm'; -import { UserData } from '../../user/models/user-data/user-data.entity'; -import { UserDataService } from '../../user/models/user-data/user-data.service'; -import { Setup2faDto, TfaType } from '../dto/output/setup-2fa.dto'; - -const TfaValidityHours = 24; -const TfaMaxTryCount = 5; - -interface SecretCacheEntry { - type: TfaType; - secret: string; - expiryDate: Date; - tryCount: number; -} - -export enum TfaLevel { - BASIC = 'Basic', - STRICT = 'Strict', -} - -@Injectable() -export class TfaService { - private readonly logger = new DfxLogger(TfaService); - - private readonly secretCache: Map = new Map(); - - constructor( - private readonly tfaRepo: TfaLogRepository, - @Inject(forwardRef(() => UserDataService)) private readonly userDataService: UserDataService, - private readonly notificationService: NotificationService, - ) {} - - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.TFA_CACHE }) - processCleanupSecretCache() { - const now = new Date(); - - const keysToBeDeleted = Array.from(this.secretCache.entries()) - .filter(([_, v]) => v.expiryDate < now) - .map(([k, _]) => k); - - keysToBeDeleted.forEach((k) => this.secretCache.delete(k)); - } - - async setup(kycHash: string, level: TfaLevel, allowStaffEnrollment = false): Promise { - const user = await this.getUser(kycHash); - if (user.isBlockedOrDeactivated) throw new ForbiddenException('Account is blocked/deactivated'); - - // Staff (Compliance/Support/RealUnit) are forced onto an app/TOTP factor: a mail code goes to the same - // inbox as the magic-link login, so it would not be an independent second factor. Everyone else keeps the - // existing mail-vs-app selection. - if (user.mail && !user.isStaff && (level === TfaLevel.BASIC || user.users.length > 0)) { - // mail 2FA - const type = TfaType.MAIL; - const secret = Util.randomIdString(6); - const codeExpiryMinutes = 30; - - this.secretCache.set(user.id, { - type, - secret, - expiryDate: Util.minutesAfter(codeExpiryMinutes), - tryCount: 0, - }); - - // send mail - await this.sendVerificationMail(user, secret, codeExpiryMinutes, MailContext.VERIFICATION_MAIL); - - return { type }; - } else { - // app 2FA - if (user.totpSecret) throw new ConflictException('2FA already set up'); - - // Initial staff enrollment must originate from a trusted (wallet-signature) session: a code-header or - // mail-elevated session shares the magic-link inbox and would not be an independent second factor. - if (user.isStaff && !allowStaffEnrollment) - throw new ForbiddenException('Staff 2FA must be enrolled from a wallet-authenticated session'); - - const type = TfaType.APP; - const { secret, uri } = generateSecret({ name: 'DFX.swiss', account: user.mail ?? '' }); - - this.secretCache.set(user.id, { - type, - secret, - expiryDate: Util.hoursAfter(3), - tryCount: 0, - }); - - return { type, secret, uri }; - } - } - - async verify(kycHash: string, token: string, ip: string, allowStaffEnrollment = false): Promise { - const user = await this.getUser(kycHash); - - let level: TfaLevel; - let type: TfaType; - - const cacheEntry = this.secretCache.get(user.id); - - if (cacheEntry?.tryCount >= TfaMaxTryCount) { - this.secretCache.delete(user.id); - throw new ForbiddenException('Invalid or expired 2FA token'); - } - - try { - if (cacheEntry?.type === TfaType.MAIL) { - if (token !== cacheEntry.secret) throw new ForbiddenException('Invalid or expired 2FA token'); - - level = user.users.length > 0 ? TfaLevel.STRICT : TfaLevel.BASIC; - type = TfaType.MAIL; - } else { - const secret = user.totpSecret ?? cacheEntry?.secret; - if (!secret) throw new NotFoundException('2FA not set up'); - - // Durable per-account lockout: an enrolled account (totpSecret set) has no secretCache entry, so the - // transient tryCount gate above never fires for it — this is the brute-force gap being closed here. - if (user.totpBlockedUntil && user.totpBlockedUntil > new Date()) - throw new ForbiddenException('Too many failed 2FA attempts, please try again later'); - - await this.verifyTotpOrLock(user, secret, token); - - if (!user.totpSecret) { - // Initial staff enrollment must originate from a trusted (wallet-signature) session; a code-header or - // mail-elevated session shares the magic-link inbox and is not an independent factor. - if (user.isStaff && !allowStaffEnrollment) - throw new ForbiddenException('Staff 2FA must be enrolled from a wallet-authenticated session'); - - await this.userDataService.updateTotpSecret(user, secret); - } - - level = TfaLevel.STRICT; - type = TfaType.APP; - } - } catch (e) { - if (cacheEntry) cacheEntry.tryCount++; - - throw e; - } - - this.secretCache.delete(user.id); - await this.createTfaLog(user, ip, level, type); - } - - private async verifyTotpOrLock(user: UserData, secret: string, token: string): Promise { - try { - this.verifyOrThrow(secret, token); - } catch (e) { - const failedAttempts = (user.totpFailedAttempts ?? 0) + 1; - const isLocked = failedAttempts >= TfaMaxTryCount; - - if (isLocked) - this.logger.warn(`TOTP lockout triggered for account ${user.id} after ${failedAttempts} failed attempts`); - - await this.userDataService.setTotpLockout( - user, - isLocked ? 0 : failedAttempts, - isLocked ? Util.minutesAfter(15) : null, - ); - - throw e; - } - - // reset the durable counter for a legitimate user so failures never accumulate to a lockout over time - if (user.totpFailedAttempts || user.totpBlockedUntil) await this.userDataService.setTotpLockout(user, 0, null); - } - - async check(userDataId: number, ip: string, level?: TfaLevel): Promise { - const userData = await this.userDataService.getUserData(userDataId, { users: true }); - if (!userData) throw new NotFoundException('User data not found'); - - await this.checkVerification(userData, ip, level, userData.isStaff); - } - - async checkVerification(user: UserData, ip: string, level?: TfaLevel, requireApp = false) { - const allowedLevels = level === TfaLevel.STRICT ? [TfaLevel.STRICT] : [TfaLevel.BASIC, TfaLevel.STRICT]; - const logs = await this.tfaRepo.findBy({ - userData: { id: user.id }, - ipAddress: ip, - created: MoreThan(Util.hoursBefore(TfaValidityHours)), - }); - - const isVerified = logs.some((log) => { - const levelOk = allowedLevels.some((l) => log.comment.includes(l)); - // Staff must have verified with an app/TOTP factor; a mail-code log never satisfies a staff check. - const typeOk = !requireApp || log.comment.includes(TfaType.APP); - // Legacy untyped 'Verified' logs predate typed logs; never accept them for a STRICT or staff check. - const legacyOk = !requireApp && level !== TfaLevel.STRICT && log.comment === 'Verified'; - return (levelOk && typeOk) || legacyOk; - }); - if (!isVerified) throw new TfaRequiredException(level); - } - - // --- HELPER METHODS --- // - async sendVerificationMail( - userData: UserData, - code: string, - expirationMinutes: number, - context: MailContext.VERIFICATION_MAIL | MailContext.EMAIL_VERIFICATION, - ): Promise { - try { - const tag = context === MailContext.VERIFICATION_MAIL ? 'default' : 'email'; - - if (userData.mail) - await this.notificationService.sendMail({ - type: MailType.USER_V2, - context, - input: { - userData: userData, - title: `${MailTranslationKey.VERIFICATION_CODE}.${tag}.title`, - salutation: { - key: `${MailTranslationKey.VERIFICATION_CODE}.${tag}.salutation`, - }, - texts: [ - { - key: `${MailTranslationKey.VERIFICATION_CODE}.message`, - params: { code }, - }, - { key: MailKey.SPACE, params: { value: '2' } }, - { - key: `${MailTranslationKey.VERIFICATION_CODE}.closing`, - params: { expiration: `${expirationMinutes}` }, - }, - { key: MailKey.SPACE, params: { value: '4' } }, - { key: MailKey.DFX_TEAM_CLOSING }, - ], - }, - }); - } catch (e) { - this.logger.error(`Failed to send verification mail ${userData.id}:`, e); - throw new ServiceUnavailableException('Failed to send verification mail'); - } - } - - private verifyOrThrow(secret: string, token: string): void { - const result = verifyToken(secret, token); - if (!result || ![0, -1].includes(result.delta)) { - this.logger.verbose(`2FA verify failed, ${!result ? 'token mismatch' : 'delta is ' + result.delta}`); - throw new ForbiddenException('Invalid or expired 2FA token'); - } - } - - private async createTfaLog(userData: UserData, ipAddress: string, level: TfaLevel, type: TfaType) { - const logEntity = this.tfaRepo.create({ - ipAddress, - userData, - comment: `${level} (${type})`, - }); - - await this.tfaRepo.save(logEntity); - } - - private async getUser(kycHash: string): Promise { - return this.userDataService.getByKycHashOrThrow(kycHash, { users: true }); - } -} +import { + ConflictException, + ForbiddenException, + Inject, + Injectable, + NotFoundException, + ServiceUnavailableException, + forwardRef, +} from '@nestjs/common'; +import { TfaRequiredException } from '../exceptions/tfa-required.exception'; +import { CronExpression } from '@nestjs/schedule'; +import { generateSecret, verifyToken } from 'node-2fa'; +import { DfxLogger } from 'src/shared/services/dfx-logger'; +import { Process } from 'src/shared/services/process.service'; +import { DfxCron } from 'src/shared/utils/cron'; +import { Util } from 'src/shared/utils/util'; +import { TfaLogRepository } from 'src/subdomains/generic/kyc/repositories/tfa-log.repository'; +import { MailContext, MailType } from 'src/subdomains/supporting/notification/enums'; +import { MailKey, MailTranslationKey } from 'src/subdomains/supporting/notification/factories/mail.factory'; +import { NotificationService } from 'src/subdomains/supporting/notification/services/notification.service'; +import { MoreThan } from 'typeorm'; +import { UserData } from '../../user/models/user-data/user-data.entity'; +import { UserDataService } from '../../user/models/user-data/user-data.service'; +import { Setup2faDto, TfaType } from '../dto/output/setup-2fa.dto'; + +const TfaValidityHours = 24; +const TfaMaxTryCount = 5; + +interface SecretCacheEntry { + type: TfaType; + secret: string; + expiryDate: Date; + tryCount: number; +} + +export enum TfaLevel { + BASIC = 'Basic', + STRICT = 'Strict', +} + +@Injectable() +export class TfaService { + private readonly logger = new DfxLogger(TfaService); + + private readonly secretCache: Map = new Map(); + + constructor( + private readonly tfaRepo: TfaLogRepository, + @Inject(forwardRef(() => UserDataService)) private readonly userDataService: UserDataService, + private readonly notificationService: NotificationService, + ) {} + + @DfxCron(CronExpression.EVERY_MINUTE, { perInstance: true, process: Process.TFA_CACHE }) + processCleanupSecretCache() { + const now = new Date(); + + const keysToBeDeleted = Array.from(this.secretCache.entries()) + .filter(([_, v]) => v.expiryDate < now) + .map(([k, _]) => k); + + keysToBeDeleted.forEach((k) => this.secretCache.delete(k)); + } + + async setup(kycHash: string, level: TfaLevel, allowStaffEnrollment = false): Promise { + const user = await this.getUser(kycHash); + if (user.isBlockedOrDeactivated) throw new ForbiddenException('Account is blocked/deactivated'); + + // Staff (Compliance/Support/RealUnit) are forced onto an app/TOTP factor: a mail code goes to the same + // inbox as the magic-link login, so it would not be an independent second factor. Everyone else keeps the + // existing mail-vs-app selection. + if (user.mail && !user.isStaff && (level === TfaLevel.BASIC || user.users.length > 0)) { + // mail 2FA + const type = TfaType.MAIL; + const secret = Util.randomIdString(6); + const codeExpiryMinutes = 30; + + this.secretCache.set(user.id, { + type, + secret, + expiryDate: Util.minutesAfter(codeExpiryMinutes), + tryCount: 0, + }); + + // send mail + await this.sendVerificationMail(user, secret, codeExpiryMinutes, MailContext.VERIFICATION_MAIL); + + return { type }; + } else { + // app 2FA + if (user.totpSecret) throw new ConflictException('2FA already set up'); + + // Initial staff enrollment must originate from a trusted (wallet-signature) session: a code-header or + // mail-elevated session shares the magic-link inbox and would not be an independent second factor. + if (user.isStaff && !allowStaffEnrollment) + throw new ForbiddenException('Staff 2FA must be enrolled from a wallet-authenticated session'); + + const type = TfaType.APP; + const { secret, uri } = generateSecret({ name: 'DFX.swiss', account: user.mail ?? '' }); + + this.secretCache.set(user.id, { + type, + secret, + expiryDate: Util.hoursAfter(3), + tryCount: 0, + }); + + return { type, secret, uri }; + } + } + + async verify(kycHash: string, token: string, ip: string, allowStaffEnrollment = false): Promise { + const user = await this.getUser(kycHash); + + let level: TfaLevel; + let type: TfaType; + + const cacheEntry = this.secretCache.get(user.id); + + if (cacheEntry?.tryCount >= TfaMaxTryCount) { + this.secretCache.delete(user.id); + throw new ForbiddenException('Invalid or expired 2FA token'); + } + + try { + if (cacheEntry?.type === TfaType.MAIL) { + if (token !== cacheEntry.secret) throw new ForbiddenException('Invalid or expired 2FA token'); + + level = user.users.length > 0 ? TfaLevel.STRICT : TfaLevel.BASIC; + type = TfaType.MAIL; + } else { + const secret = user.totpSecret ?? cacheEntry?.secret; + if (!secret) throw new NotFoundException('2FA not set up'); + + // Durable per-account lockout: an enrolled account (totpSecret set) has no secretCache entry, so the + // transient tryCount gate above never fires for it — this is the brute-force gap being closed here. + if (user.totpBlockedUntil && user.totpBlockedUntil > new Date()) + throw new ForbiddenException('Too many failed 2FA attempts, please try again later'); + + await this.verifyTotpOrLock(user, secret, token); + + if (!user.totpSecret) { + // Initial staff enrollment must originate from a trusted (wallet-signature) session; a code-header or + // mail-elevated session shares the magic-link inbox and is not an independent factor. + if (user.isStaff && !allowStaffEnrollment) + throw new ForbiddenException('Staff 2FA must be enrolled from a wallet-authenticated session'); + + await this.userDataService.updateTotpSecret(user, secret); + } + + level = TfaLevel.STRICT; + type = TfaType.APP; + } + } catch (e) { + if (cacheEntry) cacheEntry.tryCount++; + + throw e; + } + + this.secretCache.delete(user.id); + await this.createTfaLog(user, ip, level, type); + } + + private async verifyTotpOrLock(user: UserData, secret: string, token: string): Promise { + try { + this.verifyOrThrow(secret, token); + } catch (e) { + const failedAttempts = (user.totpFailedAttempts ?? 0) + 1; + const isLocked = failedAttempts >= TfaMaxTryCount; + + if (isLocked) + this.logger.warn(`TOTP lockout triggered for account ${user.id} after ${failedAttempts} failed attempts`); + + await this.userDataService.setTotpLockout( + user, + isLocked ? 0 : failedAttempts, + isLocked ? Util.minutesAfter(15) : null, + ); + + throw e; + } + + // reset the durable counter for a legitimate user so failures never accumulate to a lockout over time + if (user.totpFailedAttempts || user.totpBlockedUntil) await this.userDataService.setTotpLockout(user, 0, null); + } + + async check(userDataId: number, ip: string, level?: TfaLevel): Promise { + const userData = await this.userDataService.getUserData(userDataId, { users: true }); + if (!userData) throw new NotFoundException('User data not found'); + + await this.checkVerification(userData, ip, level, userData.isStaff); + } + + async checkVerification(user: UserData, ip: string, level?: TfaLevel, requireApp = false) { + const allowedLevels = level === TfaLevel.STRICT ? [TfaLevel.STRICT] : [TfaLevel.BASIC, TfaLevel.STRICT]; + const logs = await this.tfaRepo.findBy({ + userData: { id: user.id }, + ipAddress: ip, + created: MoreThan(Util.hoursBefore(TfaValidityHours)), + }); + + const isVerified = logs.some((log) => { + const levelOk = allowedLevels.some((l) => log.comment.includes(l)); + // Staff must have verified with an app/TOTP factor; a mail-code log never satisfies a staff check. + const typeOk = !requireApp || log.comment.includes(TfaType.APP); + // Legacy untyped 'Verified' logs predate typed logs; never accept them for a STRICT or staff check. + const legacyOk = !requireApp && level !== TfaLevel.STRICT && log.comment === 'Verified'; + return (levelOk && typeOk) || legacyOk; + }); + if (!isVerified) throw new TfaRequiredException(level); + } + + // --- HELPER METHODS --- // + async sendVerificationMail( + userData: UserData, + code: string, + expirationMinutes: number, + context: MailContext.VERIFICATION_MAIL | MailContext.EMAIL_VERIFICATION, + ): Promise { + try { + const tag = context === MailContext.VERIFICATION_MAIL ? 'default' : 'email'; + + if (userData.mail) + await this.notificationService.sendMail({ + type: MailType.USER_V2, + context, + input: { + userData: userData, + title: `${MailTranslationKey.VERIFICATION_CODE}.${tag}.title`, + salutation: { + key: `${MailTranslationKey.VERIFICATION_CODE}.${tag}.salutation`, + }, + texts: [ + { + key: `${MailTranslationKey.VERIFICATION_CODE}.message`, + params: { code }, + }, + { key: MailKey.SPACE, params: { value: '2' } }, + { + key: `${MailTranslationKey.VERIFICATION_CODE}.closing`, + params: { expiration: `${expirationMinutes}` }, + }, + { key: MailKey.SPACE, params: { value: '4' } }, + { key: MailKey.DFX_TEAM_CLOSING }, + ], + }, + }); + } catch (e) { + this.logger.error(`Failed to send verification mail ${userData.id}:`, e); + throw new ServiceUnavailableException('Failed to send verification mail'); + } + } + + private verifyOrThrow(secret: string, token: string): void { + const result = verifyToken(secret, token); + if (!result || ![0, -1].includes(result.delta)) { + this.logger.verbose(`2FA verify failed, ${!result ? 'token mismatch' : 'delta is ' + result.delta}`); + throw new ForbiddenException('Invalid or expired 2FA token'); + } + } + + private async createTfaLog(userData: UserData, ipAddress: string, level: TfaLevel, type: TfaType) { + const logEntity = this.tfaRepo.create({ + ipAddress, + userData, + comment: `${level} (${type})`, + }); + + await this.tfaRepo.save(logEntity); + } + + private async getUser(kycHash: string): Promise { + return this.userDataService.getByKycHashOrThrow(kycHash, { users: true }); + } +} diff --git a/src/subdomains/generic/user/models/auth/auth-lnurl.service.ts b/src/subdomains/generic/user/models/auth/auth-lnurl.service.ts index 4eb286e26b..1f81569fe9 100644 --- a/src/subdomains/generic/user/models/auth/auth-lnurl.service.ts +++ b/src/subdomains/generic/user/models/auth/auth-lnurl.service.ts @@ -1,159 +1,159 @@ -import { ForbiddenException, Injectable, NotFoundException } from '@nestjs/common'; -import { CronExpression } from '@nestjs/schedule'; -import { secp256k1 } from '@noble/curves/secp256k1'; -import { randomBytes } from 'crypto'; -import { Config } from 'src/config/config'; -import { LightningHelper } from 'src/integration/lightning/lightning-helper'; -import { IpLogService } from 'src/shared/models/ip-log/ip-log.service'; -import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; -import { Util } from 'src/shared/utils/util'; -import { AuthService } from 'src/subdomains/generic/user/models/auth/auth.service'; -import { - AuthLnurlCreateLoginResponseDto, - AuthLnurlResponseStatus, - AuthLnurlSignInResponseDto, - AuthLnurlSignupDto, - AuthLnurlStatusResponseDto, -} from 'src/subdomains/generic/user/models/auth/dto/auth-lnurl.dto'; -import { WalletType } from '../user/user.enum'; - -export interface AuthCacheDto { - servicesIp: string; - servicesUrl: string; - k1: string; - k1CreationTime: number; - accessToken?: string; - accessTokenCreationTime?: number; -} - -@Injectable() -export class AuthLnUrlService { - private readonly authCache: Map = new Map(); - - constructor( - private readonly authService: AuthService, - private readonly ipLogService: IpLogService, - ) {} - - @DfxCron(CronExpression.EVERY_30_SECONDS, { process: Process.LNURL_AUTH_CACHE }) - processCleanupAccessToken() { - const before30SecTime = Util.secondsBefore(30).getTime(); - - const keysToBeDeleted = [...this.authCache.entries()] - .filter((k) => k[1].accessTokenCreationTime < before30SecTime) - .map((k) => k[0]); - - keysToBeDeleted.forEach((k) => this.authCache.delete(k)); - } - - @DfxCron(CronExpression.EVERY_5_MINUTES, { process: Process.LNURL_AUTH_CACHE }) - processCleanupAuthCache() { - const before5MinTime = Util.minutesBefore(5).getTime(); - - const keysToBeDeleted = [...this.authCache.entries()] - .filter((k) => k[1].k1CreationTime < before5MinTime) - .map((k) => k[0]); - - keysToBeDeleted.forEach((k) => this.authCache.delete(k)); - } - - create(servicesIp: string, servicesUrl: string): AuthLnurlCreateLoginResponseDto { - const k1 = Util.createHash(randomBytes(32)); - - this.authCache.set(k1, { - servicesIp: servicesIp, - servicesUrl: servicesUrl, - k1: k1, - k1CreationTime: Date.now(), - }); - - const url = new URL(`${Config.url()}/lnurla`); - url.searchParams.set('tag', 'login'); - url.searchParams.set('action', 'login'); - url.searchParams.set('k1', k1); - - return { k1: k1, lnurl: LightningHelper.encodeLnurl(url.toString()) }; - } - - async login(signupDto: AuthLnurlSignupDto, userIp: string): Promise { - const checkSignupResponse = this.checkSignupDto(signupDto); - - if (checkSignupResponse) { - this.authCache.delete(signupDto.k1); - return checkSignupResponse; - } - - const { k1, sig, key, address } = signupDto; - - const authCacheEntry = this.authCache.get(k1); - const { servicesIp, servicesUrl } = authCacheEntry; - - const ipLog = await this.ipLogService.create(servicesIp, servicesUrl, address, WalletType.DFX_TARO); - - if (!ipLog.result) { - this.authCache.delete(k1); - throw new ForbiddenException('The country of IP address is not allowed'); - } - - try { - const verifyResult = secp256k1.verify( - Util.stringToUint8(sig, 'hex'), - Util.stringToUint8(k1, 'hex'), - Util.stringToUint8(key, 'hex'), - ); - if (!verifyResult) return AuthLnurlSignInResponseDto.createError('invalid auth signature'); - - authCacheEntry.accessToken = await this.signIn(signupDto, servicesIp, userIp); - authCacheEntry.accessTokenCreationTime = Date.now(); - - return AuthLnurlSignInResponseDto.createOk(); - } catch (e) { - return { status: AuthLnurlResponseStatus.ERROR, reason: e.message ?? 'invalid signup' }; - } - } - - private checkSignupDto(signupDto: AuthLnurlSignupDto): AuthLnurlSignInResponseDto | undefined { - if ('login' !== signupDto.tag) return AuthLnurlSignInResponseDto.createError('invalid tag'); - if ('login' !== signupDto.action) return AuthLnurlSignInResponseDto.createError('invalid action'); - - const authCacheEntry = this.authCache.get(signupDto.k1); - if (!authCacheEntry) return AuthLnurlSignInResponseDto.createError('invalid challenge'); - - const checkBeforeTime = Util.minutesBefore(5).getTime(); - if (authCacheEntry.k1CreationTime < checkBeforeTime) - return AuthLnurlSignInResponseDto.createError('challenge expired'); - } - - async signIn(signupDto: AuthLnurlSignupDto, servicesIp: string, userIp: string): Promise { - const session = { address: signupDto.address, signature: signupDto.signature, walletType: WalletType.DFX_TARO }; - - const { accessToken } = await this.authService.signIn(session, userIp, true).catch((e) => { - if (e instanceof NotFoundException) - return this.authService.signUp( - { - ...session, - usedRef: signupDto.usedRef, - wallet: signupDto.wallet ?? 'DFX Bitcoin', - recommendationCode: signupDto.recommendationCode, - }, - servicesIp, - ); - throw e; - }); - - return accessToken; - } - - status(k1: string): AuthLnurlStatusResponseDto { - const authCacheEntry = this.authCache.get(k1); - if (!authCacheEntry) throw new NotFoundException('k1 not found'); - - const accessToken = authCacheEntry.accessToken; - if (!accessToken) return { isComplete: false }; - - this.authCache.delete(k1); - - return { isComplete: true, accessToken: accessToken }; - } -} +import { ForbiddenException, Injectable, NotFoundException } from '@nestjs/common'; +import { CronExpression } from '@nestjs/schedule'; +import { secp256k1 } from '@noble/curves/secp256k1'; +import { randomBytes } from 'crypto'; +import { Config } from 'src/config/config'; +import { LightningHelper } from 'src/integration/lightning/lightning-helper'; +import { IpLogService } from 'src/shared/models/ip-log/ip-log.service'; +import { Process } from 'src/shared/services/process.service'; +import { DfxCron } from 'src/shared/utils/cron'; +import { Util } from 'src/shared/utils/util'; +import { AuthService } from 'src/subdomains/generic/user/models/auth/auth.service'; +import { + AuthLnurlCreateLoginResponseDto, + AuthLnurlResponseStatus, + AuthLnurlSignInResponseDto, + AuthLnurlSignupDto, + AuthLnurlStatusResponseDto, +} from 'src/subdomains/generic/user/models/auth/dto/auth-lnurl.dto'; +import { WalletType } from '../user/user.enum'; + +export interface AuthCacheDto { + servicesIp: string; + servicesUrl: string; + k1: string; + k1CreationTime: number; + accessToken?: string; + accessTokenCreationTime?: number; +} + +@Injectable() +export class AuthLnUrlService { + private readonly authCache: Map = new Map(); + + constructor( + private readonly authService: AuthService, + private readonly ipLogService: IpLogService, + ) {} + + @DfxCron(CronExpression.EVERY_30_SECONDS, { perInstance: true, process: Process.LNURL_AUTH_CACHE }) + processCleanupAccessToken() { + const before30SecTime = Util.secondsBefore(30).getTime(); + + const keysToBeDeleted = [...this.authCache.entries()] + .filter((k) => k[1].accessTokenCreationTime < before30SecTime) + .map((k) => k[0]); + + keysToBeDeleted.forEach((k) => this.authCache.delete(k)); + } + + @DfxCron(CronExpression.EVERY_5_MINUTES, { perInstance: true, process: Process.LNURL_AUTH_CACHE }) + processCleanupAuthCache() { + const before5MinTime = Util.minutesBefore(5).getTime(); + + const keysToBeDeleted = [...this.authCache.entries()] + .filter((k) => k[1].k1CreationTime < before5MinTime) + .map((k) => k[0]); + + keysToBeDeleted.forEach((k) => this.authCache.delete(k)); + } + + create(servicesIp: string, servicesUrl: string): AuthLnurlCreateLoginResponseDto { + const k1 = Util.createHash(randomBytes(32)); + + this.authCache.set(k1, { + servicesIp: servicesIp, + servicesUrl: servicesUrl, + k1: k1, + k1CreationTime: Date.now(), + }); + + const url = new URL(`${Config.url()}/lnurla`); + url.searchParams.set('tag', 'login'); + url.searchParams.set('action', 'login'); + url.searchParams.set('k1', k1); + + return { k1: k1, lnurl: LightningHelper.encodeLnurl(url.toString()) }; + } + + async login(signupDto: AuthLnurlSignupDto, userIp: string): Promise { + const checkSignupResponse = this.checkSignupDto(signupDto); + + if (checkSignupResponse) { + this.authCache.delete(signupDto.k1); + return checkSignupResponse; + } + + const { k1, sig, key, address } = signupDto; + + const authCacheEntry = this.authCache.get(k1); + const { servicesIp, servicesUrl } = authCacheEntry; + + const ipLog = await this.ipLogService.create(servicesIp, servicesUrl, address, WalletType.DFX_TARO); + + if (!ipLog.result) { + this.authCache.delete(k1); + throw new ForbiddenException('The country of IP address is not allowed'); + } + + try { + const verifyResult = secp256k1.verify( + Util.stringToUint8(sig, 'hex'), + Util.stringToUint8(k1, 'hex'), + Util.stringToUint8(key, 'hex'), + ); + if (!verifyResult) return AuthLnurlSignInResponseDto.createError('invalid auth signature'); + + authCacheEntry.accessToken = await this.signIn(signupDto, servicesIp, userIp); + authCacheEntry.accessTokenCreationTime = Date.now(); + + return AuthLnurlSignInResponseDto.createOk(); + } catch (e) { + return { status: AuthLnurlResponseStatus.ERROR, reason: e.message ?? 'invalid signup' }; + } + } + + private checkSignupDto(signupDto: AuthLnurlSignupDto): AuthLnurlSignInResponseDto | undefined { + if ('login' !== signupDto.tag) return AuthLnurlSignInResponseDto.createError('invalid tag'); + if ('login' !== signupDto.action) return AuthLnurlSignInResponseDto.createError('invalid action'); + + const authCacheEntry = this.authCache.get(signupDto.k1); + if (!authCacheEntry) return AuthLnurlSignInResponseDto.createError('invalid challenge'); + + const checkBeforeTime = Util.minutesBefore(5).getTime(); + if (authCacheEntry.k1CreationTime < checkBeforeTime) + return AuthLnurlSignInResponseDto.createError('challenge expired'); + } + + async signIn(signupDto: AuthLnurlSignupDto, servicesIp: string, userIp: string): Promise { + const session = { address: signupDto.address, signature: signupDto.signature, walletType: WalletType.DFX_TARO }; + + const { accessToken } = await this.authService.signIn(session, userIp, true).catch((e) => { + if (e instanceof NotFoundException) + return this.authService.signUp( + { + ...session, + usedRef: signupDto.usedRef, + wallet: signupDto.wallet ?? 'DFX Bitcoin', + recommendationCode: signupDto.recommendationCode, + }, + servicesIp, + ); + throw e; + }); + + return accessToken; + } + + status(k1: string): AuthLnurlStatusResponseDto { + const authCacheEntry = this.authCache.get(k1); + if (!authCacheEntry) throw new NotFoundException('k1 not found'); + + const accessToken = authCacheEntry.accessToken; + if (!accessToken) return { isComplete: false }; + + this.authCache.delete(k1); + + return { isComplete: true, accessToken: accessToken }; + } +} From 461d2f87074c125fb477efe0ecba8e87cac73fc7 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:15 +0200 Subject: [PATCH 07/86] Stop the Spark token optimization on HTTP-only instances The Spark client starts a bare setInterval in its constructor that runs optimizeTokenOutputs against the wallet every five minutes. It predates the scheduler and bypasses every switch, so with the process split both containers would drive on-chain maintenance against the same seed. It is global work and belongs to the job instance, so it now honours the same flag as the cron jobs. This was the last remaining path by which periodic work could run outside DfxCronService. --- src/integration/blockchain/spark/spark-client.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/integration/blockchain/spark/spark-client.ts b/src/integration/blockchain/spark/spark-client.ts index cd06ccdff0..22d136e5fd 100644 --- a/src/integration/blockchain/spark/spark-client.ts +++ b/src/integration/blockchain/spark/spark-client.ts @@ -226,6 +226,11 @@ export class SparkClient extends BlockchainClient { } private startTokenOptimization(): void { + // On-chain wallet maintenance is global work: it must run on exactly one instance. + // This timer predates the scheduler and bypasses it, so an HTTP-only instance would + // otherwise drive optimizeTokenOutputs against the same seed as the job instance. + if (!GetConfig().cronJobsEnabled) return; + if (this.tokenOptimizationInterval) clearInterval(this.tokenOptimizationInterval); const intervalMs = 5 * 60 * 1000; // 5 minutes From 0d136123164ac909ae98e056faa1392d6b8ab2af Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:17 +0200 Subject: [PATCH 08/86] Restore CRLF line endings and mark the pool monitors per-instance Three files were rewritten with LF by tooling, turning one-line changes into roughly 1100 lines of diff noise and guaranteed merge conflicts for any parallel branch touching them. The repository mixes both endings and has no .gitattributes, so the original ending is restored rather than normalised here. monitorConnectionPool and monitorConnectionPoolStatic measure this process's own pool and only log. They are harmless to run everywhere by construction, and without them pool visibility would be lost on exactly the instance whose pool serves the requests - the same reasoning that put perInstance on monitorEventLoop. --- .../monitor-connection-pool.service.ts | 96 +-- .../services/payment-link-fee.service.ts | 230 ++++---- .../generic/kyc/services/tfa.service.ts | 548 +++++++++--------- .../user/models/auth/auth-lnurl.service.ts | 318 +++++----- 4 files changed, 596 insertions(+), 596 deletions(-) diff --git a/src/subdomains/core/monitoring/monitor-connection-pool.service.ts b/src/subdomains/core/monitoring/monitor-connection-pool.service.ts index 9ba51b4611..b5215ec590 100644 --- a/src/subdomains/core/monitoring/monitor-connection-pool.service.ts +++ b/src/subdomains/core/monitoring/monitor-connection-pool.service.ts @@ -1,48 +1,48 @@ -import { Injectable } from '@nestjs/common'; -import { CronExpression } from '@nestjs/schedule'; -import { Config } from 'src/config/config'; -import { DfxLogger } from 'src/shared/services/dfx-logger'; -import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; -import { DataSource } from 'typeorm'; -import { PostgresConnectionOptions } from 'typeorm/driver/postgres/PostgresConnectionOptions'; -import { PostgresDriver } from 'typeorm/driver/postgres/PostgresDriver'; - -@Injectable() -export class MonitorConnectionPoolService { - private readonly logger = new DfxLogger(MonitorConnectionPoolService); - - private readonly dbConnectionPool: any; // pg.Pool - - constructor(dataSource: DataSource) { - const dbDriver = dataSource.driver as PostgresDriver; - this.dbConnectionPool = dbDriver.master; - } - - @DfxCron(CronExpression.EVERY_SECOND, { process: Process.MONITOR_CONNECTION_POOL }) - monitorConnectionPool() { - const dbOptions = Config.database as PostgresConnectionOptions; - const dbMaxPoolConnections = dbOptions.poolSize ?? 10; - - const total = this.dbConnectionPool.totalCount; - const idle = this.dbConnectionPool.idleCount; - const waiting = this.dbConnectionPool.waitingCount; - - if (dbMaxPoolConnections === total && idle === 0) { - // Warning, if there are all connections in use - this.logger.warn(`ConnectionPool with max. borrowed connections: T${total}/I${idle}/W${waiting}`); - } else if (waiting > 0) { - // Info, if there is a pending connection - this.logger.info(`ConnectionPool with pending connections: T${total}/I${idle}/W${waiting}`); - } - } - - @DfxCron(CronExpression.EVERY_10_SECONDS, { process: Process.MONITOR_CONNECTION_POOL }) - monitorConnectionPoolStatic() { - const total = this.dbConnectionPool.totalCount; - const idle = this.dbConnectionPool.idleCount; - const waiting = this.dbConnectionPool.waitingCount; - - this.logger.info(`ConnectionPool connections: T${total}/I${idle}/W${waiting}`); - } -} +import { Injectable } from '@nestjs/common'; +import { CronExpression } from '@nestjs/schedule'; +import { Config } from 'src/config/config'; +import { DfxLogger } from 'src/shared/services/dfx-logger'; +import { Process } from 'src/shared/services/process.service'; +import { DfxCron } from 'src/shared/utils/cron'; +import { DataSource } from 'typeorm'; +import { PostgresConnectionOptions } from 'typeorm/driver/postgres/PostgresConnectionOptions'; +import { PostgresDriver } from 'typeorm/driver/postgres/PostgresDriver'; + +@Injectable() +export class MonitorConnectionPoolService { + private readonly logger = new DfxLogger(MonitorConnectionPoolService); + + private readonly dbConnectionPool: any; // pg.Pool + + constructor(dataSource: DataSource) { + const dbDriver = dataSource.driver as PostgresDriver; + this.dbConnectionPool = dbDriver.master; + } + + @DfxCron(CronExpression.EVERY_SECOND, { perInstance: true, process: Process.MONITOR_CONNECTION_POOL }) + monitorConnectionPool() { + const dbOptions = Config.database as PostgresConnectionOptions; + const dbMaxPoolConnections = dbOptions.poolSize ?? 10; + + const total = this.dbConnectionPool.totalCount; + const idle = this.dbConnectionPool.idleCount; + const waiting = this.dbConnectionPool.waitingCount; + + if (dbMaxPoolConnections === total && idle === 0) { + // Warning, if there are all connections in use + this.logger.warn(`ConnectionPool with max. borrowed connections: T${total}/I${idle}/W${waiting}`); + } else if (waiting > 0) { + // Info, if there is a pending connection + this.logger.info(`ConnectionPool with pending connections: T${total}/I${idle}/W${waiting}`); + } + } + + @DfxCron(CronExpression.EVERY_10_SECONDS, { perInstance: true, process: Process.MONITOR_CONNECTION_POOL }) + monitorConnectionPoolStatic() { + const total = this.dbConnectionPool.totalCount; + const idle = this.dbConnectionPool.idleCount; + const waiting = this.dbConnectionPool.waitingCount; + + this.logger.info(`ConnectionPool connections: T${total}/I${idle}/W${waiting}`); + } +} diff --git a/src/subdomains/core/payment-link/services/payment-link-fee.service.ts b/src/subdomains/core/payment-link/services/payment-link-fee.service.ts index 8bcc8e2391..242e37070a 100644 --- a/src/subdomains/core/payment-link/services/payment-link-fee.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-fee.service.ts @@ -1,115 +1,115 @@ -import { Injectable, OnModuleInit } from '@nestjs/common'; -import { CronExpression } from '@nestjs/schedule'; -import { Environment, GetConfig } from 'src/config/config'; -import { MIN_FEE_RATE_SAT_VB } from 'src/integration/blockchain/bitcoin/services/bitcoin-based-fee.service'; -import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.enum'; -import { PaymentLinkBlockchains } from 'src/integration/blockchain/shared/util/blockchain.util'; -import { DfxLogger } from 'src/shared/services/dfx-logger'; -import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; -import { Util } from 'src/shared/utils/util'; -import { PayoutBitcoinService } from 'src/subdomains/supporting/payout/services/payout-bitcoin.service'; -import { PayoutFiroService } from 'src/subdomains/supporting/payout/services/payout-firo.service'; -import { BlockchainRegistryService } from '../../../../integration/blockchain/shared/services/blockchain-registry.service'; - -interface FeeCacheData { - timestamp: Date; - fee: number; -} - -@Injectable() -export class PaymentLinkFeeService implements OnModuleInit { - private readonly logger = new DfxLogger(PaymentLinkFeeService); - - private static readonly MINUTES_5 = 5 * 60; - - private readonly feeCache: Map; - - constructor( - private readonly blockchainRegistryService: BlockchainRegistryService, - private readonly payoutBitcoinService: PayoutBitcoinService, - private readonly payoutFiroService: PayoutFiroService, - ) { - this.feeCache = new Map(); - } - - onModuleInit() { - void this.updateFees(); - } - - // --- JOBS --- // - @DfxCron(CronExpression.EVERY_MINUTE, { perInstance: true, process: Process.UPDATE_BLOCKCHAIN_FEE }) - async updateFees(): Promise { - if (GetConfig().environment === Environment.LOC) return; - - for (const blockchain of PaymentLinkBlockchains) { - try { - const fee = await this.calculateFee(blockchain); - this.feeCache.set(blockchain, { - timestamp: new Date(), - fee, - }); - } catch (e) { - this.feeCache.delete(blockchain); - this.logger.error(`Failed to get fee for blockchain ${blockchain}:`, e); - } - } - } - - private async calculateFee(blockchain: Blockchain): Promise { - switch (blockchain) { - case Blockchain.BINANCE_PAY: - case Blockchain.KUCOIN_PAY: - case Blockchain.LIGHTNING: - case Blockchain.MONERO: - case Blockchain.ZANO: - case Blockchain.SOLANA: - case Blockchain.TRON: - case Blockchain.CARDANO: - case Blockchain.INTERNET_COMPUTER: - return 0; - - case Blockchain.ETHEREUM: - case Blockchain.SEPOLIA: - case Blockchain.ARBITRUM: - case Blockchain.OPTIMISM: - case Blockchain.BASE: - case Blockchain.GNOSIS: - case Blockchain.POLYGON: - case Blockchain.BINANCE_SMART_CHAIN: { - const client = this.blockchainRegistryService.getEvmClient(blockchain); - return +(await client.getRecommendedGasPrice()); - } - - // The customer minimum is the network's own minimum for an inbound payment to confirm — it - // must NOT include the CPFP/default margin from getSendFeeRate, which exists only for DFX's - // own outbound spends. The value differs per chain because the chains do, but neither carries - // the payout margin. - case Blockchain.BITCOIN: - // Bitcoin fees are user-adjustable and the chain can congest, so use the recommended - // (next-block) rate, which adapts to congestion — floored at the relay minimum so the - // advertised minimum is always relayable. - return Math.max(await this.payoutBitcoinService.getRecommendedFeeRate(), MIN_FEE_RATE_SAT_VB); - - case Blockchain.FIRO: - // Same principle as Bitcoin: Firo's own next-block rate without the payout margin, floored - // at the relay minimum so it stays relayable. The current OCP deposit address is transparent, - // so a Stack Wallet payment is a Spark-spend to it, whose fee sits at the relay floor and - // cannot be raised; Firo does not congest and its node usually returns no estimate, so this - // resolves to the relay floor in practice — exactly what that Spark-spend pays. A dedicated - // relay-floor cap belongs here only once a Spark `sm1…` deposit address is deployed, whose - // protocol-capped fee cannot follow a congestion-adaptive minimum. - return Math.max(await this.payoutFiroService.getRecommendedFeeRate(), MIN_FEE_RATE_SAT_VB); - } - } - - // --- PUBLIC METHODS --- // - async getMinFee(blockchain: Blockchain): Promise { - const cacheData = this.feeCache.get(blockchain); - if (!cacheData) return; - - if (Util.secondsDiff(cacheData.timestamp) > PaymentLinkFeeService.MINUTES_5) return; - - return cacheData.fee; - } -} +import { Injectable, OnModuleInit } from '@nestjs/common'; +import { CronExpression } from '@nestjs/schedule'; +import { Environment, GetConfig } from 'src/config/config'; +import { MIN_FEE_RATE_SAT_VB } from 'src/integration/blockchain/bitcoin/services/bitcoin-based-fee.service'; +import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.enum'; +import { PaymentLinkBlockchains } from 'src/integration/blockchain/shared/util/blockchain.util'; +import { DfxLogger } from 'src/shared/services/dfx-logger'; +import { Process } from 'src/shared/services/process.service'; +import { DfxCron } from 'src/shared/utils/cron'; +import { Util } from 'src/shared/utils/util'; +import { PayoutBitcoinService } from 'src/subdomains/supporting/payout/services/payout-bitcoin.service'; +import { PayoutFiroService } from 'src/subdomains/supporting/payout/services/payout-firo.service'; +import { BlockchainRegistryService } from '../../../../integration/blockchain/shared/services/blockchain-registry.service'; + +interface FeeCacheData { + timestamp: Date; + fee: number; +} + +@Injectable() +export class PaymentLinkFeeService implements OnModuleInit { + private readonly logger = new DfxLogger(PaymentLinkFeeService); + + private static readonly MINUTES_5 = 5 * 60; + + private readonly feeCache: Map; + + constructor( + private readonly blockchainRegistryService: BlockchainRegistryService, + private readonly payoutBitcoinService: PayoutBitcoinService, + private readonly payoutFiroService: PayoutFiroService, + ) { + this.feeCache = new Map(); + } + + onModuleInit() { + void this.updateFees(); + } + + // --- JOBS --- // + @DfxCron(CronExpression.EVERY_MINUTE, { perInstance: true, process: Process.UPDATE_BLOCKCHAIN_FEE }) + async updateFees(): Promise { + if (GetConfig().environment === Environment.LOC) return; + + for (const blockchain of PaymentLinkBlockchains) { + try { + const fee = await this.calculateFee(blockchain); + this.feeCache.set(blockchain, { + timestamp: new Date(), + fee, + }); + } catch (e) { + this.feeCache.delete(blockchain); + this.logger.error(`Failed to get fee for blockchain ${blockchain}:`, e); + } + } + } + + private async calculateFee(blockchain: Blockchain): Promise { + switch (blockchain) { + case Blockchain.BINANCE_PAY: + case Blockchain.KUCOIN_PAY: + case Blockchain.LIGHTNING: + case Blockchain.MONERO: + case Blockchain.ZANO: + case Blockchain.SOLANA: + case Blockchain.TRON: + case Blockchain.CARDANO: + case Blockchain.INTERNET_COMPUTER: + return 0; + + case Blockchain.ETHEREUM: + case Blockchain.SEPOLIA: + case Blockchain.ARBITRUM: + case Blockchain.OPTIMISM: + case Blockchain.BASE: + case Blockchain.GNOSIS: + case Blockchain.POLYGON: + case Blockchain.BINANCE_SMART_CHAIN: { + const client = this.blockchainRegistryService.getEvmClient(blockchain); + return +(await client.getRecommendedGasPrice()); + } + + // The customer minimum is the network's own minimum for an inbound payment to confirm — it + // must NOT include the CPFP/default margin from getSendFeeRate, which exists only for DFX's + // own outbound spends. The value differs per chain because the chains do, but neither carries + // the payout margin. + case Blockchain.BITCOIN: + // Bitcoin fees are user-adjustable and the chain can congest, so use the recommended + // (next-block) rate, which adapts to congestion — floored at the relay minimum so the + // advertised minimum is always relayable. + return Math.max(await this.payoutBitcoinService.getRecommendedFeeRate(), MIN_FEE_RATE_SAT_VB); + + case Blockchain.FIRO: + // Same principle as Bitcoin: Firo's own next-block rate without the payout margin, floored + // at the relay minimum so it stays relayable. The current OCP deposit address is transparent, + // so a Stack Wallet payment is a Spark-spend to it, whose fee sits at the relay floor and + // cannot be raised; Firo does not congest and its node usually returns no estimate, so this + // resolves to the relay floor in practice — exactly what that Spark-spend pays. A dedicated + // relay-floor cap belongs here only once a Spark `sm1…` deposit address is deployed, whose + // protocol-capped fee cannot follow a congestion-adaptive minimum. + return Math.max(await this.payoutFiroService.getRecommendedFeeRate(), MIN_FEE_RATE_SAT_VB); + } + } + + // --- PUBLIC METHODS --- // + async getMinFee(blockchain: Blockchain): Promise { + const cacheData = this.feeCache.get(blockchain); + if (!cacheData) return; + + if (Util.secondsDiff(cacheData.timestamp) > PaymentLinkFeeService.MINUTES_5) return; + + return cacheData.fee; + } +} diff --git a/src/subdomains/generic/kyc/services/tfa.service.ts b/src/subdomains/generic/kyc/services/tfa.service.ts index e84636072c..3f5795ea19 100644 --- a/src/subdomains/generic/kyc/services/tfa.service.ts +++ b/src/subdomains/generic/kyc/services/tfa.service.ts @@ -1,274 +1,274 @@ -import { - ConflictException, - ForbiddenException, - Inject, - Injectable, - NotFoundException, - ServiceUnavailableException, - forwardRef, -} from '@nestjs/common'; -import { TfaRequiredException } from '../exceptions/tfa-required.exception'; -import { CronExpression } from '@nestjs/schedule'; -import { generateSecret, verifyToken } from 'node-2fa'; -import { DfxLogger } from 'src/shared/services/dfx-logger'; -import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; -import { Util } from 'src/shared/utils/util'; -import { TfaLogRepository } from 'src/subdomains/generic/kyc/repositories/tfa-log.repository'; -import { MailContext, MailType } from 'src/subdomains/supporting/notification/enums'; -import { MailKey, MailTranslationKey } from 'src/subdomains/supporting/notification/factories/mail.factory'; -import { NotificationService } from 'src/subdomains/supporting/notification/services/notification.service'; -import { MoreThan } from 'typeorm'; -import { UserData } from '../../user/models/user-data/user-data.entity'; -import { UserDataService } from '../../user/models/user-data/user-data.service'; -import { Setup2faDto, TfaType } from '../dto/output/setup-2fa.dto'; - -const TfaValidityHours = 24; -const TfaMaxTryCount = 5; - -interface SecretCacheEntry { - type: TfaType; - secret: string; - expiryDate: Date; - tryCount: number; -} - -export enum TfaLevel { - BASIC = 'Basic', - STRICT = 'Strict', -} - -@Injectable() -export class TfaService { - private readonly logger = new DfxLogger(TfaService); - - private readonly secretCache: Map = new Map(); - - constructor( - private readonly tfaRepo: TfaLogRepository, - @Inject(forwardRef(() => UserDataService)) private readonly userDataService: UserDataService, - private readonly notificationService: NotificationService, - ) {} - - @DfxCron(CronExpression.EVERY_MINUTE, { perInstance: true, process: Process.TFA_CACHE }) - processCleanupSecretCache() { - const now = new Date(); - - const keysToBeDeleted = Array.from(this.secretCache.entries()) - .filter(([_, v]) => v.expiryDate < now) - .map(([k, _]) => k); - - keysToBeDeleted.forEach((k) => this.secretCache.delete(k)); - } - - async setup(kycHash: string, level: TfaLevel, allowStaffEnrollment = false): Promise { - const user = await this.getUser(kycHash); - if (user.isBlockedOrDeactivated) throw new ForbiddenException('Account is blocked/deactivated'); - - // Staff (Compliance/Support/RealUnit) are forced onto an app/TOTP factor: a mail code goes to the same - // inbox as the magic-link login, so it would not be an independent second factor. Everyone else keeps the - // existing mail-vs-app selection. - if (user.mail && !user.isStaff && (level === TfaLevel.BASIC || user.users.length > 0)) { - // mail 2FA - const type = TfaType.MAIL; - const secret = Util.randomIdString(6); - const codeExpiryMinutes = 30; - - this.secretCache.set(user.id, { - type, - secret, - expiryDate: Util.minutesAfter(codeExpiryMinutes), - tryCount: 0, - }); - - // send mail - await this.sendVerificationMail(user, secret, codeExpiryMinutes, MailContext.VERIFICATION_MAIL); - - return { type }; - } else { - // app 2FA - if (user.totpSecret) throw new ConflictException('2FA already set up'); - - // Initial staff enrollment must originate from a trusted (wallet-signature) session: a code-header or - // mail-elevated session shares the magic-link inbox and would not be an independent second factor. - if (user.isStaff && !allowStaffEnrollment) - throw new ForbiddenException('Staff 2FA must be enrolled from a wallet-authenticated session'); - - const type = TfaType.APP; - const { secret, uri } = generateSecret({ name: 'DFX.swiss', account: user.mail ?? '' }); - - this.secretCache.set(user.id, { - type, - secret, - expiryDate: Util.hoursAfter(3), - tryCount: 0, - }); - - return { type, secret, uri }; - } - } - - async verify(kycHash: string, token: string, ip: string, allowStaffEnrollment = false): Promise { - const user = await this.getUser(kycHash); - - let level: TfaLevel; - let type: TfaType; - - const cacheEntry = this.secretCache.get(user.id); - - if (cacheEntry?.tryCount >= TfaMaxTryCount) { - this.secretCache.delete(user.id); - throw new ForbiddenException('Invalid or expired 2FA token'); - } - - try { - if (cacheEntry?.type === TfaType.MAIL) { - if (token !== cacheEntry.secret) throw new ForbiddenException('Invalid or expired 2FA token'); - - level = user.users.length > 0 ? TfaLevel.STRICT : TfaLevel.BASIC; - type = TfaType.MAIL; - } else { - const secret = user.totpSecret ?? cacheEntry?.secret; - if (!secret) throw new NotFoundException('2FA not set up'); - - // Durable per-account lockout: an enrolled account (totpSecret set) has no secretCache entry, so the - // transient tryCount gate above never fires for it — this is the brute-force gap being closed here. - if (user.totpBlockedUntil && user.totpBlockedUntil > new Date()) - throw new ForbiddenException('Too many failed 2FA attempts, please try again later'); - - await this.verifyTotpOrLock(user, secret, token); - - if (!user.totpSecret) { - // Initial staff enrollment must originate from a trusted (wallet-signature) session; a code-header or - // mail-elevated session shares the magic-link inbox and is not an independent factor. - if (user.isStaff && !allowStaffEnrollment) - throw new ForbiddenException('Staff 2FA must be enrolled from a wallet-authenticated session'); - - await this.userDataService.updateTotpSecret(user, secret); - } - - level = TfaLevel.STRICT; - type = TfaType.APP; - } - } catch (e) { - if (cacheEntry) cacheEntry.tryCount++; - - throw e; - } - - this.secretCache.delete(user.id); - await this.createTfaLog(user, ip, level, type); - } - - private async verifyTotpOrLock(user: UserData, secret: string, token: string): Promise { - try { - this.verifyOrThrow(secret, token); - } catch (e) { - const failedAttempts = (user.totpFailedAttempts ?? 0) + 1; - const isLocked = failedAttempts >= TfaMaxTryCount; - - if (isLocked) - this.logger.warn(`TOTP lockout triggered for account ${user.id} after ${failedAttempts} failed attempts`); - - await this.userDataService.setTotpLockout( - user, - isLocked ? 0 : failedAttempts, - isLocked ? Util.minutesAfter(15) : null, - ); - - throw e; - } - - // reset the durable counter for a legitimate user so failures never accumulate to a lockout over time - if (user.totpFailedAttempts || user.totpBlockedUntil) await this.userDataService.setTotpLockout(user, 0, null); - } - - async check(userDataId: number, ip: string, level?: TfaLevel): Promise { - const userData = await this.userDataService.getUserData(userDataId, { users: true }); - if (!userData) throw new NotFoundException('User data not found'); - - await this.checkVerification(userData, ip, level, userData.isStaff); - } - - async checkVerification(user: UserData, ip: string, level?: TfaLevel, requireApp = false) { - const allowedLevels = level === TfaLevel.STRICT ? [TfaLevel.STRICT] : [TfaLevel.BASIC, TfaLevel.STRICT]; - const logs = await this.tfaRepo.findBy({ - userData: { id: user.id }, - ipAddress: ip, - created: MoreThan(Util.hoursBefore(TfaValidityHours)), - }); - - const isVerified = logs.some((log) => { - const levelOk = allowedLevels.some((l) => log.comment.includes(l)); - // Staff must have verified with an app/TOTP factor; a mail-code log never satisfies a staff check. - const typeOk = !requireApp || log.comment.includes(TfaType.APP); - // Legacy untyped 'Verified' logs predate typed logs; never accept them for a STRICT or staff check. - const legacyOk = !requireApp && level !== TfaLevel.STRICT && log.comment === 'Verified'; - return (levelOk && typeOk) || legacyOk; - }); - if (!isVerified) throw new TfaRequiredException(level); - } - - // --- HELPER METHODS --- // - async sendVerificationMail( - userData: UserData, - code: string, - expirationMinutes: number, - context: MailContext.VERIFICATION_MAIL | MailContext.EMAIL_VERIFICATION, - ): Promise { - try { - const tag = context === MailContext.VERIFICATION_MAIL ? 'default' : 'email'; - - if (userData.mail) - await this.notificationService.sendMail({ - type: MailType.USER_V2, - context, - input: { - userData: userData, - title: `${MailTranslationKey.VERIFICATION_CODE}.${tag}.title`, - salutation: { - key: `${MailTranslationKey.VERIFICATION_CODE}.${tag}.salutation`, - }, - texts: [ - { - key: `${MailTranslationKey.VERIFICATION_CODE}.message`, - params: { code }, - }, - { key: MailKey.SPACE, params: { value: '2' } }, - { - key: `${MailTranslationKey.VERIFICATION_CODE}.closing`, - params: { expiration: `${expirationMinutes}` }, - }, - { key: MailKey.SPACE, params: { value: '4' } }, - { key: MailKey.DFX_TEAM_CLOSING }, - ], - }, - }); - } catch (e) { - this.logger.error(`Failed to send verification mail ${userData.id}:`, e); - throw new ServiceUnavailableException('Failed to send verification mail'); - } - } - - private verifyOrThrow(secret: string, token: string): void { - const result = verifyToken(secret, token); - if (!result || ![0, -1].includes(result.delta)) { - this.logger.verbose(`2FA verify failed, ${!result ? 'token mismatch' : 'delta is ' + result.delta}`); - throw new ForbiddenException('Invalid or expired 2FA token'); - } - } - - private async createTfaLog(userData: UserData, ipAddress: string, level: TfaLevel, type: TfaType) { - const logEntity = this.tfaRepo.create({ - ipAddress, - userData, - comment: `${level} (${type})`, - }); - - await this.tfaRepo.save(logEntity); - } - - private async getUser(kycHash: string): Promise { - return this.userDataService.getByKycHashOrThrow(kycHash, { users: true }); - } -} +import { + ConflictException, + ForbiddenException, + Inject, + Injectable, + NotFoundException, + ServiceUnavailableException, + forwardRef, +} from '@nestjs/common'; +import { TfaRequiredException } from '../exceptions/tfa-required.exception'; +import { CronExpression } from '@nestjs/schedule'; +import { generateSecret, verifyToken } from 'node-2fa'; +import { DfxLogger } from 'src/shared/services/dfx-logger'; +import { Process } from 'src/shared/services/process.service'; +import { DfxCron } from 'src/shared/utils/cron'; +import { Util } from 'src/shared/utils/util'; +import { TfaLogRepository } from 'src/subdomains/generic/kyc/repositories/tfa-log.repository'; +import { MailContext, MailType } from 'src/subdomains/supporting/notification/enums'; +import { MailKey, MailTranslationKey } from 'src/subdomains/supporting/notification/factories/mail.factory'; +import { NotificationService } from 'src/subdomains/supporting/notification/services/notification.service'; +import { MoreThan } from 'typeorm'; +import { UserData } from '../../user/models/user-data/user-data.entity'; +import { UserDataService } from '../../user/models/user-data/user-data.service'; +import { Setup2faDto, TfaType } from '../dto/output/setup-2fa.dto'; + +const TfaValidityHours = 24; +const TfaMaxTryCount = 5; + +interface SecretCacheEntry { + type: TfaType; + secret: string; + expiryDate: Date; + tryCount: number; +} + +export enum TfaLevel { + BASIC = 'Basic', + STRICT = 'Strict', +} + +@Injectable() +export class TfaService { + private readonly logger = new DfxLogger(TfaService); + + private readonly secretCache: Map = new Map(); + + constructor( + private readonly tfaRepo: TfaLogRepository, + @Inject(forwardRef(() => UserDataService)) private readonly userDataService: UserDataService, + private readonly notificationService: NotificationService, + ) {} + + @DfxCron(CronExpression.EVERY_MINUTE, { perInstance: true, process: Process.TFA_CACHE }) + processCleanupSecretCache() { + const now = new Date(); + + const keysToBeDeleted = Array.from(this.secretCache.entries()) + .filter(([_, v]) => v.expiryDate < now) + .map(([k, _]) => k); + + keysToBeDeleted.forEach((k) => this.secretCache.delete(k)); + } + + async setup(kycHash: string, level: TfaLevel, allowStaffEnrollment = false): Promise { + const user = await this.getUser(kycHash); + if (user.isBlockedOrDeactivated) throw new ForbiddenException('Account is blocked/deactivated'); + + // Staff (Compliance/Support/RealUnit) are forced onto an app/TOTP factor: a mail code goes to the same + // inbox as the magic-link login, so it would not be an independent second factor. Everyone else keeps the + // existing mail-vs-app selection. + if (user.mail && !user.isStaff && (level === TfaLevel.BASIC || user.users.length > 0)) { + // mail 2FA + const type = TfaType.MAIL; + const secret = Util.randomIdString(6); + const codeExpiryMinutes = 30; + + this.secretCache.set(user.id, { + type, + secret, + expiryDate: Util.minutesAfter(codeExpiryMinutes), + tryCount: 0, + }); + + // send mail + await this.sendVerificationMail(user, secret, codeExpiryMinutes, MailContext.VERIFICATION_MAIL); + + return { type }; + } else { + // app 2FA + if (user.totpSecret) throw new ConflictException('2FA already set up'); + + // Initial staff enrollment must originate from a trusted (wallet-signature) session: a code-header or + // mail-elevated session shares the magic-link inbox and would not be an independent second factor. + if (user.isStaff && !allowStaffEnrollment) + throw new ForbiddenException('Staff 2FA must be enrolled from a wallet-authenticated session'); + + const type = TfaType.APP; + const { secret, uri } = generateSecret({ name: 'DFX.swiss', account: user.mail ?? '' }); + + this.secretCache.set(user.id, { + type, + secret, + expiryDate: Util.hoursAfter(3), + tryCount: 0, + }); + + return { type, secret, uri }; + } + } + + async verify(kycHash: string, token: string, ip: string, allowStaffEnrollment = false): Promise { + const user = await this.getUser(kycHash); + + let level: TfaLevel; + let type: TfaType; + + const cacheEntry = this.secretCache.get(user.id); + + if (cacheEntry?.tryCount >= TfaMaxTryCount) { + this.secretCache.delete(user.id); + throw new ForbiddenException('Invalid or expired 2FA token'); + } + + try { + if (cacheEntry?.type === TfaType.MAIL) { + if (token !== cacheEntry.secret) throw new ForbiddenException('Invalid or expired 2FA token'); + + level = user.users.length > 0 ? TfaLevel.STRICT : TfaLevel.BASIC; + type = TfaType.MAIL; + } else { + const secret = user.totpSecret ?? cacheEntry?.secret; + if (!secret) throw new NotFoundException('2FA not set up'); + + // Durable per-account lockout: an enrolled account (totpSecret set) has no secretCache entry, so the + // transient tryCount gate above never fires for it — this is the brute-force gap being closed here. + if (user.totpBlockedUntil && user.totpBlockedUntil > new Date()) + throw new ForbiddenException('Too many failed 2FA attempts, please try again later'); + + await this.verifyTotpOrLock(user, secret, token); + + if (!user.totpSecret) { + // Initial staff enrollment must originate from a trusted (wallet-signature) session; a code-header or + // mail-elevated session shares the magic-link inbox and is not an independent factor. + if (user.isStaff && !allowStaffEnrollment) + throw new ForbiddenException('Staff 2FA must be enrolled from a wallet-authenticated session'); + + await this.userDataService.updateTotpSecret(user, secret); + } + + level = TfaLevel.STRICT; + type = TfaType.APP; + } + } catch (e) { + if (cacheEntry) cacheEntry.tryCount++; + + throw e; + } + + this.secretCache.delete(user.id); + await this.createTfaLog(user, ip, level, type); + } + + private async verifyTotpOrLock(user: UserData, secret: string, token: string): Promise { + try { + this.verifyOrThrow(secret, token); + } catch (e) { + const failedAttempts = (user.totpFailedAttempts ?? 0) + 1; + const isLocked = failedAttempts >= TfaMaxTryCount; + + if (isLocked) + this.logger.warn(`TOTP lockout triggered for account ${user.id} after ${failedAttempts} failed attempts`); + + await this.userDataService.setTotpLockout( + user, + isLocked ? 0 : failedAttempts, + isLocked ? Util.minutesAfter(15) : null, + ); + + throw e; + } + + // reset the durable counter for a legitimate user so failures never accumulate to a lockout over time + if (user.totpFailedAttempts || user.totpBlockedUntil) await this.userDataService.setTotpLockout(user, 0, null); + } + + async check(userDataId: number, ip: string, level?: TfaLevel): Promise { + const userData = await this.userDataService.getUserData(userDataId, { users: true }); + if (!userData) throw new NotFoundException('User data not found'); + + await this.checkVerification(userData, ip, level, userData.isStaff); + } + + async checkVerification(user: UserData, ip: string, level?: TfaLevel, requireApp = false) { + const allowedLevels = level === TfaLevel.STRICT ? [TfaLevel.STRICT] : [TfaLevel.BASIC, TfaLevel.STRICT]; + const logs = await this.tfaRepo.findBy({ + userData: { id: user.id }, + ipAddress: ip, + created: MoreThan(Util.hoursBefore(TfaValidityHours)), + }); + + const isVerified = logs.some((log) => { + const levelOk = allowedLevels.some((l) => log.comment.includes(l)); + // Staff must have verified with an app/TOTP factor; a mail-code log never satisfies a staff check. + const typeOk = !requireApp || log.comment.includes(TfaType.APP); + // Legacy untyped 'Verified' logs predate typed logs; never accept them for a STRICT or staff check. + const legacyOk = !requireApp && level !== TfaLevel.STRICT && log.comment === 'Verified'; + return (levelOk && typeOk) || legacyOk; + }); + if (!isVerified) throw new TfaRequiredException(level); + } + + // --- HELPER METHODS --- // + async sendVerificationMail( + userData: UserData, + code: string, + expirationMinutes: number, + context: MailContext.VERIFICATION_MAIL | MailContext.EMAIL_VERIFICATION, + ): Promise { + try { + const tag = context === MailContext.VERIFICATION_MAIL ? 'default' : 'email'; + + if (userData.mail) + await this.notificationService.sendMail({ + type: MailType.USER_V2, + context, + input: { + userData: userData, + title: `${MailTranslationKey.VERIFICATION_CODE}.${tag}.title`, + salutation: { + key: `${MailTranslationKey.VERIFICATION_CODE}.${tag}.salutation`, + }, + texts: [ + { + key: `${MailTranslationKey.VERIFICATION_CODE}.message`, + params: { code }, + }, + { key: MailKey.SPACE, params: { value: '2' } }, + { + key: `${MailTranslationKey.VERIFICATION_CODE}.closing`, + params: { expiration: `${expirationMinutes}` }, + }, + { key: MailKey.SPACE, params: { value: '4' } }, + { key: MailKey.DFX_TEAM_CLOSING }, + ], + }, + }); + } catch (e) { + this.logger.error(`Failed to send verification mail ${userData.id}:`, e); + throw new ServiceUnavailableException('Failed to send verification mail'); + } + } + + private verifyOrThrow(secret: string, token: string): void { + const result = verifyToken(secret, token); + if (!result || ![0, -1].includes(result.delta)) { + this.logger.verbose(`2FA verify failed, ${!result ? 'token mismatch' : 'delta is ' + result.delta}`); + throw new ForbiddenException('Invalid or expired 2FA token'); + } + } + + private async createTfaLog(userData: UserData, ipAddress: string, level: TfaLevel, type: TfaType) { + const logEntity = this.tfaRepo.create({ + ipAddress, + userData, + comment: `${level} (${type})`, + }); + + await this.tfaRepo.save(logEntity); + } + + private async getUser(kycHash: string): Promise { + return this.userDataService.getByKycHashOrThrow(kycHash, { users: true }); + } +} diff --git a/src/subdomains/generic/user/models/auth/auth-lnurl.service.ts b/src/subdomains/generic/user/models/auth/auth-lnurl.service.ts index 1f81569fe9..d0fbfdf8dd 100644 --- a/src/subdomains/generic/user/models/auth/auth-lnurl.service.ts +++ b/src/subdomains/generic/user/models/auth/auth-lnurl.service.ts @@ -1,159 +1,159 @@ -import { ForbiddenException, Injectable, NotFoundException } from '@nestjs/common'; -import { CronExpression } from '@nestjs/schedule'; -import { secp256k1 } from '@noble/curves/secp256k1'; -import { randomBytes } from 'crypto'; -import { Config } from 'src/config/config'; -import { LightningHelper } from 'src/integration/lightning/lightning-helper'; -import { IpLogService } from 'src/shared/models/ip-log/ip-log.service'; -import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; -import { Util } from 'src/shared/utils/util'; -import { AuthService } from 'src/subdomains/generic/user/models/auth/auth.service'; -import { - AuthLnurlCreateLoginResponseDto, - AuthLnurlResponseStatus, - AuthLnurlSignInResponseDto, - AuthLnurlSignupDto, - AuthLnurlStatusResponseDto, -} from 'src/subdomains/generic/user/models/auth/dto/auth-lnurl.dto'; -import { WalletType } from '../user/user.enum'; - -export interface AuthCacheDto { - servicesIp: string; - servicesUrl: string; - k1: string; - k1CreationTime: number; - accessToken?: string; - accessTokenCreationTime?: number; -} - -@Injectable() -export class AuthLnUrlService { - private readonly authCache: Map = new Map(); - - constructor( - private readonly authService: AuthService, - private readonly ipLogService: IpLogService, - ) {} - - @DfxCron(CronExpression.EVERY_30_SECONDS, { perInstance: true, process: Process.LNURL_AUTH_CACHE }) - processCleanupAccessToken() { - const before30SecTime = Util.secondsBefore(30).getTime(); - - const keysToBeDeleted = [...this.authCache.entries()] - .filter((k) => k[1].accessTokenCreationTime < before30SecTime) - .map((k) => k[0]); - - keysToBeDeleted.forEach((k) => this.authCache.delete(k)); - } - - @DfxCron(CronExpression.EVERY_5_MINUTES, { perInstance: true, process: Process.LNURL_AUTH_CACHE }) - processCleanupAuthCache() { - const before5MinTime = Util.minutesBefore(5).getTime(); - - const keysToBeDeleted = [...this.authCache.entries()] - .filter((k) => k[1].k1CreationTime < before5MinTime) - .map((k) => k[0]); - - keysToBeDeleted.forEach((k) => this.authCache.delete(k)); - } - - create(servicesIp: string, servicesUrl: string): AuthLnurlCreateLoginResponseDto { - const k1 = Util.createHash(randomBytes(32)); - - this.authCache.set(k1, { - servicesIp: servicesIp, - servicesUrl: servicesUrl, - k1: k1, - k1CreationTime: Date.now(), - }); - - const url = new URL(`${Config.url()}/lnurla`); - url.searchParams.set('tag', 'login'); - url.searchParams.set('action', 'login'); - url.searchParams.set('k1', k1); - - return { k1: k1, lnurl: LightningHelper.encodeLnurl(url.toString()) }; - } - - async login(signupDto: AuthLnurlSignupDto, userIp: string): Promise { - const checkSignupResponse = this.checkSignupDto(signupDto); - - if (checkSignupResponse) { - this.authCache.delete(signupDto.k1); - return checkSignupResponse; - } - - const { k1, sig, key, address } = signupDto; - - const authCacheEntry = this.authCache.get(k1); - const { servicesIp, servicesUrl } = authCacheEntry; - - const ipLog = await this.ipLogService.create(servicesIp, servicesUrl, address, WalletType.DFX_TARO); - - if (!ipLog.result) { - this.authCache.delete(k1); - throw new ForbiddenException('The country of IP address is not allowed'); - } - - try { - const verifyResult = secp256k1.verify( - Util.stringToUint8(sig, 'hex'), - Util.stringToUint8(k1, 'hex'), - Util.stringToUint8(key, 'hex'), - ); - if (!verifyResult) return AuthLnurlSignInResponseDto.createError('invalid auth signature'); - - authCacheEntry.accessToken = await this.signIn(signupDto, servicesIp, userIp); - authCacheEntry.accessTokenCreationTime = Date.now(); - - return AuthLnurlSignInResponseDto.createOk(); - } catch (e) { - return { status: AuthLnurlResponseStatus.ERROR, reason: e.message ?? 'invalid signup' }; - } - } - - private checkSignupDto(signupDto: AuthLnurlSignupDto): AuthLnurlSignInResponseDto | undefined { - if ('login' !== signupDto.tag) return AuthLnurlSignInResponseDto.createError('invalid tag'); - if ('login' !== signupDto.action) return AuthLnurlSignInResponseDto.createError('invalid action'); - - const authCacheEntry = this.authCache.get(signupDto.k1); - if (!authCacheEntry) return AuthLnurlSignInResponseDto.createError('invalid challenge'); - - const checkBeforeTime = Util.minutesBefore(5).getTime(); - if (authCacheEntry.k1CreationTime < checkBeforeTime) - return AuthLnurlSignInResponseDto.createError('challenge expired'); - } - - async signIn(signupDto: AuthLnurlSignupDto, servicesIp: string, userIp: string): Promise { - const session = { address: signupDto.address, signature: signupDto.signature, walletType: WalletType.DFX_TARO }; - - const { accessToken } = await this.authService.signIn(session, userIp, true).catch((e) => { - if (e instanceof NotFoundException) - return this.authService.signUp( - { - ...session, - usedRef: signupDto.usedRef, - wallet: signupDto.wallet ?? 'DFX Bitcoin', - recommendationCode: signupDto.recommendationCode, - }, - servicesIp, - ); - throw e; - }); - - return accessToken; - } - - status(k1: string): AuthLnurlStatusResponseDto { - const authCacheEntry = this.authCache.get(k1); - if (!authCacheEntry) throw new NotFoundException('k1 not found'); - - const accessToken = authCacheEntry.accessToken; - if (!accessToken) return { isComplete: false }; - - this.authCache.delete(k1); - - return { isComplete: true, accessToken: accessToken }; - } -} +import { ForbiddenException, Injectable, NotFoundException } from '@nestjs/common'; +import { CronExpression } from '@nestjs/schedule'; +import { secp256k1 } from '@noble/curves/secp256k1'; +import { randomBytes } from 'crypto'; +import { Config } from 'src/config/config'; +import { LightningHelper } from 'src/integration/lightning/lightning-helper'; +import { IpLogService } from 'src/shared/models/ip-log/ip-log.service'; +import { Process } from 'src/shared/services/process.service'; +import { DfxCron } from 'src/shared/utils/cron'; +import { Util } from 'src/shared/utils/util'; +import { AuthService } from 'src/subdomains/generic/user/models/auth/auth.service'; +import { + AuthLnurlCreateLoginResponseDto, + AuthLnurlResponseStatus, + AuthLnurlSignInResponseDto, + AuthLnurlSignupDto, + AuthLnurlStatusResponseDto, +} from 'src/subdomains/generic/user/models/auth/dto/auth-lnurl.dto'; +import { WalletType } from '../user/user.enum'; + +export interface AuthCacheDto { + servicesIp: string; + servicesUrl: string; + k1: string; + k1CreationTime: number; + accessToken?: string; + accessTokenCreationTime?: number; +} + +@Injectable() +export class AuthLnUrlService { + private readonly authCache: Map = new Map(); + + constructor( + private readonly authService: AuthService, + private readonly ipLogService: IpLogService, + ) {} + + @DfxCron(CronExpression.EVERY_30_SECONDS, { perInstance: true, process: Process.LNURL_AUTH_CACHE }) + processCleanupAccessToken() { + const before30SecTime = Util.secondsBefore(30).getTime(); + + const keysToBeDeleted = [...this.authCache.entries()] + .filter((k) => k[1].accessTokenCreationTime < before30SecTime) + .map((k) => k[0]); + + keysToBeDeleted.forEach((k) => this.authCache.delete(k)); + } + + @DfxCron(CronExpression.EVERY_5_MINUTES, { perInstance: true, process: Process.LNURL_AUTH_CACHE }) + processCleanupAuthCache() { + const before5MinTime = Util.minutesBefore(5).getTime(); + + const keysToBeDeleted = [...this.authCache.entries()] + .filter((k) => k[1].k1CreationTime < before5MinTime) + .map((k) => k[0]); + + keysToBeDeleted.forEach((k) => this.authCache.delete(k)); + } + + create(servicesIp: string, servicesUrl: string): AuthLnurlCreateLoginResponseDto { + const k1 = Util.createHash(randomBytes(32)); + + this.authCache.set(k1, { + servicesIp: servicesIp, + servicesUrl: servicesUrl, + k1: k1, + k1CreationTime: Date.now(), + }); + + const url = new URL(`${Config.url()}/lnurla`); + url.searchParams.set('tag', 'login'); + url.searchParams.set('action', 'login'); + url.searchParams.set('k1', k1); + + return { k1: k1, lnurl: LightningHelper.encodeLnurl(url.toString()) }; + } + + async login(signupDto: AuthLnurlSignupDto, userIp: string): Promise { + const checkSignupResponse = this.checkSignupDto(signupDto); + + if (checkSignupResponse) { + this.authCache.delete(signupDto.k1); + return checkSignupResponse; + } + + const { k1, sig, key, address } = signupDto; + + const authCacheEntry = this.authCache.get(k1); + const { servicesIp, servicesUrl } = authCacheEntry; + + const ipLog = await this.ipLogService.create(servicesIp, servicesUrl, address, WalletType.DFX_TARO); + + if (!ipLog.result) { + this.authCache.delete(k1); + throw new ForbiddenException('The country of IP address is not allowed'); + } + + try { + const verifyResult = secp256k1.verify( + Util.stringToUint8(sig, 'hex'), + Util.stringToUint8(k1, 'hex'), + Util.stringToUint8(key, 'hex'), + ); + if (!verifyResult) return AuthLnurlSignInResponseDto.createError('invalid auth signature'); + + authCacheEntry.accessToken = await this.signIn(signupDto, servicesIp, userIp); + authCacheEntry.accessTokenCreationTime = Date.now(); + + return AuthLnurlSignInResponseDto.createOk(); + } catch (e) { + return { status: AuthLnurlResponseStatus.ERROR, reason: e.message ?? 'invalid signup' }; + } + } + + private checkSignupDto(signupDto: AuthLnurlSignupDto): AuthLnurlSignInResponseDto | undefined { + if ('login' !== signupDto.tag) return AuthLnurlSignInResponseDto.createError('invalid tag'); + if ('login' !== signupDto.action) return AuthLnurlSignInResponseDto.createError('invalid action'); + + const authCacheEntry = this.authCache.get(signupDto.k1); + if (!authCacheEntry) return AuthLnurlSignInResponseDto.createError('invalid challenge'); + + const checkBeforeTime = Util.minutesBefore(5).getTime(); + if (authCacheEntry.k1CreationTime < checkBeforeTime) + return AuthLnurlSignInResponseDto.createError('challenge expired'); + } + + async signIn(signupDto: AuthLnurlSignupDto, servicesIp: string, userIp: string): Promise { + const session = { address: signupDto.address, signature: signupDto.signature, walletType: WalletType.DFX_TARO }; + + const { accessToken } = await this.authService.signIn(session, userIp, true).catch((e) => { + if (e instanceof NotFoundException) + return this.authService.signUp( + { + ...session, + usedRef: signupDto.usedRef, + wallet: signupDto.wallet ?? 'DFX Bitcoin', + recommendationCode: signupDto.recommendationCode, + }, + servicesIp, + ); + throw e; + }); + + return accessToken; + } + + status(k1: string): AuthLnurlStatusResponseDto { + const authCacheEntry = this.authCache.get(k1); + if (!authCacheEntry) throw new NotFoundException('k1 not found'); + + const accessToken = authCacheEntry.accessToken; + if (!accessToken) return { isComplete: false }; + + this.authCache.delete(k1); + + return { isComplete: true, accessToken: accessToken }; + } +} From e7fd09033926907a273a5d026e13def568fd091b Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:18 +0200 Subject: [PATCH 09/86] Document the global vs. per-instance rule where it is read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every new cron job now carries a classification decision, and getting it wrong fails silently on the HTTP instance — the JWT denylists were the first example, three more surfaced in later review rounds. The rule therefore belongs in the cron section of CONTRIBUTING, not only in the TSDoc of the flag. Also corrects a comment in runtime-metrics.ts that no longer held: monitorEventLoop became per-instance in the meantime, so it does run on an HTTP-only instance. The reason for the metric stands - a queryable series instead of a log line, and independent of how the scheduler is configured. --- CONTRIBUTING.md | 28 ++++++++++++++++++++++++++++ src/runtime-metrics.ts | 8 ++++---- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 262cd00f6d..66fe9f8a36 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -539,6 +539,34 @@ runs unconditionally and cannot be switched off without a deploy. Prefer longer intervals (15min) over aggressive polling (1min). Only use short intervals when truly needed. +#### Global vs. per-instance + +The API can run as more than one instance from the same image — one serving HTTP, one running +the jobs (`CRON_JOBS_ENABLED=false` keeps an instance HTTP-only). **Every new cron job needs a +decision which of the two it is**, and getting it wrong fails silently: + +```typescript +// GLOBAL (the default): writes to the database, moves money, calls an external system in a +// way that changes state. Runs on the job instance only. +@DfxCron(CronExpression.EVERY_MINUTE, { process: Process.PAYMENT }) +async processPayments(): Promise {} + +// PER-INSTANCE: effect confined to this process — refreshing an in-memory copy of global +// state, expiring a local cache, measuring this process. Runs everywhere, including the +// HTTP-only instance, because requests on that instance read what it maintains. +@DfxCron(CronExpression.EVERY_30_SECONDS, { perInstance: true }) +async resyncDeniedJwtAccounts(): Promise {} +``` + +Ask: *does an HTTP handler on this instance read state that this job writes?* If yes, it is +per-instance — otherwise that state freezes at boot wherever the job does not run. The JWT +denylists are the cautionary example: frozen, they fail open and a blocked account keeps its +live tokens. + +Running a per-instance job twice must be harmless by construction. If it writes to the +database, sends mail, or calls a paid external API, it is global — mark it as such and, if an +HTTP path needs its result, route that result through the database rather than process memory. + ### Await Discipline ```typescript diff --git a/src/runtime-metrics.ts b/src/runtime-metrics.ts index b426476869..3c01996528 100644 --- a/src/runtime-metrics.ts +++ b/src/runtime-metrics.ts @@ -8,10 +8,10 @@ import { isTelemetryEnabled } from './tracing'; // waits behind a saturated event loop looks identical to one waiting on a slow query. These // metrics close that gap: they measure whether the single JS thread had capacity at all. // -// MonitorEventLoopService logs the same figures for humans, but it runs as a cron job — an -// HTTP-only instance (CRON_JOBS_ENABLED=false) registers no jobs and would therefore report -// nothing exactly where saturation matters most. The collection here is driven by the OTel -// metric reader instead, so it is independent of the scheduler and runs on every instance. +// MonitorEventLoopService logs the same figures for humans, but only as a log line, and it +// depends on the scheduler being registered. Collection here is driven by the OTel metric +// reader instead, so it holds regardless of how the scheduler is configured — and it produces +// a queryable series rather than text that has to be parsed back out of the logs. // // Export travels the existing OTLP pipeline (see src/tracing.ts). With // OTEL_EXPORTER_OTLP_ENDPOINT unset, no meter is registered and the app boots unchanged. From e328908b90accc6ce5fc2aa54889ed57993b5d14 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:20 +0200 Subject: [PATCH 10/86] Give a process a role instead of a cron on/off switch CRON_JOBS_ENABLED could express "this process runs the jobs" and "this process runs none", but not "this job belongs to the process serving HTTP". That third case exists: a job maintaining state that only a request path reads, or driving work bound to the connections a process holds open, is worse off in the worker than it is today. The flag also carried its answer in the wrong place - an optional perInstance defaulting to "global" points the silent failure at the dangerous side, because a job wrongly left unmarked stops refreshing the cache that requests on the HTTP process read, without an error anywhere. CRON_ROLE replaces it with three operating modes and CronScope with three job properties. A role runs its own scope plus `both`; `all` runs everything and is the single-process mode, which keeps local development, the test suite and every environment without a separate worker exactly as they are. The variable has no default and an unknown or empty value aborts the boot. Every default is silent in one direction: defaulting to `worker` makes a misconfigured API process run all background work a second time, defaulting to `api` makes a misconfigured worker do nothing at all. Neither raises an error, and duplicate execution of financial jobs is worse than a failed boot. The existing perInstance markings move to scope: `both` for the fifteen jobs refreshing state that both sides read, with two corrections. StatisticService becomes `api` - the hourly aggregation only feeds GET /statistic, so running it in the worker would compute a value nobody there retrieves. BinancePayService becomes `worker` - getCertificates() refreshes lazily on read, so its cache cannot go stale in a process that never runs the job. Registration also logs one line per process stating which split actually applies. A job reaching the scheduler through a dynamically resolved provider or an abstract base class is counted there and nowhere else, so the effective assignment is read from the log rather than inferred from a list. The Spark token optimization follows the same rule from its own timer: it is on-chain wallet maintenance and must run in exactly one process. --- .env.example | 9 +- .../cron-jobs-enabled.config.spec.ts | 63 ----------- src/config/__tests__/cron-role.config.spec.ts | 64 +++++++++++ src/config/config.ts | 57 ++++++---- .../services/binance-pay.service.ts | 4 +- .../blockchain/spark/spark-client.ts | 10 +- .../controllers/exchange.controller.ts | 4 +- src/jest-env.setup.ts | 6 + .../__tests__/dfx-cron.service.spec.ts | 99 ++++++++++------- src/shared/services/dfx-cron.service.ts | 46 ++++++-- src/shared/services/process.service.ts | 8 +- src/shared/utils/cron.ts | 104 +++++++++++------- .../controllers/transaction.controller.ts | 4 +- .../monitor-connection-pool.service.ts | 6 +- .../monitoring/monitor-event-loop.service.ts | 4 +- .../services/payment-link-fee.service.ts | 4 +- .../core/statistic/statistic.service.ts | 4 +- .../generic/kyc/services/tfa.service.ts | 4 +- .../user/models/auth/auth-lnurl.service.ts | 6 +- .../generic/user/models/auth/auth.service.ts | 4 +- .../models/user-data/user-data.service.ts | 4 +- .../payment/services/transaction-helper.ts | 4 +- 22 files changed, 304 insertions(+), 214 deletions(-) delete mode 100644 src/config/__tests__/cron-jobs-enabled.config.spec.ts create mode 100644 src/config/__tests__/cron-role.config.spec.ts diff --git a/.env.example b/.env.example index bce6b8b0a5..1cffa3cd38 100644 --- a/.env.example +++ b/.env.example @@ -344,7 +344,8 @@ REQUEST_KNOWN_IPS= CRON_JOB_DELAY= -# 'true' or 'false' only — an empty or misspelled value fails the boot on purpose, so that an -# instance meant to serve HTTP only never re-registers the scheduler. Omit the line entirely to -# run cron jobs, which is the default behaviour. -CRON_JOBS_ENABLED=true +# Which jobs this process registers: 'all', 'api' or 'worker'. Mandatory — a missing, empty or +# unknown value fails the boot on purpose, because every possible default lets a misconfiguration +# run silently: one process would do the background work twice, or not at all. +# 'all' is the single-process mode and the right value unless a separate worker process exists. +CRON_ROLE=all diff --git a/src/config/__tests__/cron-jobs-enabled.config.spec.ts b/src/config/__tests__/cron-jobs-enabled.config.spec.ts deleted file mode 100644 index c6b9c2ee77..0000000000 --- a/src/config/__tests__/cron-jobs-enabled.config.spec.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { Config, ConfigService, GetConfig, parseCronJobsEnabled } from '../config'; - -describe('parseCronJobsEnabled', () => { - it('defaults to enabled when the variable is absent, so existing environments keep running jobs', () => { - expect(parseCronJobsEnabled(undefined)).toBe(true); - }); - - it('accepts the two valid values', () => { - expect(parseCronJobsEnabled('true')).toBe(true); - expect(parseCronJobsEnabled('false')).toBe(false); - }); - - it.each(['', ' ', 'fals', 'False', 'FALSE', '0', 'no', 'off', 'disabled', ' false'])( - 'throws on %p instead of silently enabling the scheduler', - (value) => { - // The dangerous direction is a value being read as "enabled": on an instance meant to be - // HTTP-only that re-registers every job, and jobs without a `process` (trades, referral - // credits, volume resets) would then run on two instances at once. Cron locks are - // per-process, so nothing else would catch it. Failing the boot is the safe outcome. - // The empty string is included deliberately — an `CRON_JOBS_ENABLED=` line or an - // unresolved `${VAR}` is the likeliest accident of all. - expect(() => parseCronJobsEnabled(value)).toThrow(/expected 'true' or 'false'/); - }, - ); -}); - -describe('Config.cronJobsEnabled', () => { - const original = process.env.CRON_JOBS_ENABLED; - - afterEach(() => { - if (original == null) delete process.env.CRON_JOBS_ENABLED; - else process.env.CRON_JOBS_ENABLED = original; - - new ConfigService(GetConfig()); - }); - - // Covers the wiring env -> parseCronJobsEnabled -> Config that DfxCronService reads, - // which unit-testing the parser alone would leave unverified. - it.each([ - ['false', false], - ['true', true], - ])('maps CRON_JOBS_ENABLED=%s to %s', (value, expected) => { - process.env.CRON_JOBS_ENABLED = value; - - new ConfigService(GetConfig()); - - expect(Config.cronJobsEnabled).toBe(expected); - }); - - it('runs jobs when the variable is absent', () => { - delete process.env.CRON_JOBS_ENABLED; - - new ConfigService(GetConfig()); - - expect(Config.cronJobsEnabled).toBe(true); - }); - - it('refuses to build a configuration from an invalid value', () => { - process.env.CRON_JOBS_ENABLED = 'fals'; - - expect(() => GetConfig()).toThrow(/expected 'true' or 'false'/); - }); -}); diff --git a/src/config/__tests__/cron-role.config.spec.ts b/src/config/__tests__/cron-role.config.spec.ts new file mode 100644 index 0000000000..4bbf6b4198 --- /dev/null +++ b/src/config/__tests__/cron-role.config.spec.ts @@ -0,0 +1,64 @@ +import { Config, ConfigService, CronRole, GetConfig, parseCronRole } from '../config'; + +describe('parseCronRole', () => { + it('accepts the three roles', () => { + expect(parseCronRole('all')).toBe(CronRole.All); + expect(parseCronRole('api')).toBe(CronRole.Api); + expect(parseCronRole('worker')).toBe(CronRole.Worker); + }); + + it.each([undefined, '', ' ', 'All', 'WORKER', 'api ', 'true', 'none'])( + 'throws on %p instead of picking a role', + (value) => { + // Every possible default is silent in one direction: 'worker' would make a misconfigured + // API process run all background work a second time, 'api' would make a misconfigured + // worker do nothing at all. Neither raises an error, and duplicate execution of financial + // jobs is worse than a failed boot. The empty string is included deliberately — a + // `CRON_ROLE=` line or an unresolved `${VAR}` is the likeliest accident of all. + expect(() => parseCronRole(value)).toThrow(/expected one of all, api, worker/); + }, + ); + + it('does not accept a scope value as a role', () => { + // `both` is a property of a job, not an operating mode of a process. Accepting it here would + // blur the two axes the split depends on. + expect(() => parseCronRole('both')).toThrow(); + }); +}); + +describe('Config.cronRole', () => { + const original = process.env.CRON_ROLE; + + afterEach(() => { + if (original == null) delete process.env.CRON_ROLE; + else process.env.CRON_ROLE = original; + + new ConfigService(GetConfig()); + }); + + // Covers the wiring env -> parseCronRole -> Config that DfxCronService reads, which + // unit-testing the parser alone would leave unverified. + it.each([ + ['all', CronRole.All], + ['api', CronRole.Api], + ['worker', CronRole.Worker], + ])('maps CRON_ROLE=%s to %s', (value, expected) => { + process.env.CRON_ROLE = value; + + new ConfigService(GetConfig()); + + expect(Config.cronRole).toBe(expected); + }); + + it('refuses to build a configuration without a role', () => { + delete process.env.CRON_ROLE; + + expect(() => GetConfig()).toThrow(/expected one of all, api, worker/); + }); + + it('refuses to build a configuration from an invalid value', () => { + process.env.CRON_ROLE = 'wroker'; + + expect(() => GetConfig()).toThrow(/expected one of all, api, worker/); + }); +}); diff --git a/src/config/config.ts b/src/config/config.ts index b79bb97c5b..45af891554 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -36,6 +36,22 @@ export enum Environment { PRD = 'prd', } +/** + * Operating mode of this process, read from CRON_ROLE. + * + * `scope` on a job describes a property of the job, this describes the process running it. + * Keeping the two apart is what allows the same image to run as an HTTP process and as a + * background worker without either knowing about the other. + */ +export enum CronRole { + /** One process runs everything: local development, tests, and any deployment without a worker. */ + All = 'all', + /** Serves HTTP; runs only jobs scoped `api` or `both`. */ + Api = 'api', + /** Runs the background work; only jobs scoped `worker` or `both`. */ + Worker = 'worker', +} + export type StorageWriteMode = 'azure' | 'dual' | 's3'; export type StorageReadSource = 'azure' | 's3'; @@ -1497,13 +1513,14 @@ export class Configuration { } // Background jobs and HTTP requests share a single Node event loop, so a busy scheduler - // delays every incoming request on the same instance. Setting this to false keeps an - // instance HTTP-only: DfxCronService then registers no job at all. + // delays every incoming request on the same process. The role decides which jobs this + // process registers, which is what allows running the same image twice: once serving HTTP, + // once running the background work. // // Note this is deliberately independent of DISABLED_PROCESSES, which only skips jobs that - // declare a `process` — jobs without one would keep running and, on a second instance, - // run twice. Cron locks are per-process and do not guard across instances. - cronJobsEnabled = parseCronJobsEnabled(process.env.CRON_JOBS_ENABLED); + // declare a `process` — jobs without one would keep running and, in a second process, + // run twice. Cron locks are per-process and do not guard across processes. + cronRole = parseCronRole(process.env.CRON_ROLE); // --- HELPERS --- // disabledProcesses = () => @@ -1513,25 +1530,25 @@ export class Configuration { } /** - * Reads the CRON_JOBS_ENABLED flag. + * Reads CRON_ROLE, the operating mode of this process. + * + * There is no default, and a missing or unknown value aborts the boot. Every possible default + * lets a misconfiguration run silently: defaulting to `worker` would make a misconfigured API + * process run all background work a second time, defaulting to `api` would make a misconfigured + * worker do nothing at all. Neither produces an error, and duplicate execution of financial + * jobs is far more damaging than a failed boot. * - * Only an entirely unset variable means "run jobs", which keeps every existing environment - * working unchanged. Every other value must be exactly 'true' or 'false' and throws otherwise, - * rather than being coerced: on an instance meant to be HTTP-only, anything that silently reads - * as "enabled" re-registers the scheduler, and jobs without a declared `process` would then run - * on two instances at once. Duplicate execution of financial jobs is far more damaging than a - * failed boot. + * The empty string is rejected for the same reason, even though it is the more likely accident + * — a `CRON_ROLE=` line in an env file, or an unresolved `${VAR}`. * - * The empty string throws for that reason too, even though it is the more common accident — an - * `CRON_JOBS_ENABLED=` line in an env file, or an unresolved `${VAR}` in a compose file. Neither - * should quietly turn a job container back on. + * `all` is not a convenience value but the single-process mode: one process runs every job, + * which is what local development, the test suite and any environment without a separate worker + * need. */ -export function parseCronJobsEnabled(value?: string): boolean { - if (value == null) return true; - if (value === 'true') return true; - if (value === 'false') return false; +export function parseCronRole(value?: string): CronRole { + if (value != null && (Object.values(CronRole) as string[]).includes(value)) return value as CronRole; - throw new Error(`Invalid CRON_JOBS_ENABLED value '${value}': expected 'true' or 'false'`); + throw new Error(`Invalid CRON_ROLE value '${value ?? ''}': expected one of ${Object.values(CronRole).join(', ')}`); } function readCert(): string | undefined { diff --git a/src/integration/binance-pay/services/binance-pay.service.ts b/src/integration/binance-pay/services/binance-pay.service.ts index f7fa5371d4..bc0fe0127c 100644 --- a/src/integration/binance-pay/services/binance-pay.service.ts +++ b/src/integration/binance-pay/services/binance-pay.service.ts @@ -5,7 +5,7 @@ import { Config } from 'src/config/config'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { HttpService } from 'src/shared/services/http.service'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { TransferInfo } from 'src/subdomains/core/payment-link/dto/payment-link.dto'; import { PaymentLinkPayment } from 'src/subdomains/core/payment-link/entities/payment-link-payment.entity'; @@ -225,7 +225,7 @@ export class BinancePayService implements C2BPaymentLinkProvider { try { const headers = this.getHeaders({}); diff --git a/src/integration/blockchain/spark/spark-client.ts b/src/integration/blockchain/spark/spark-client.ts index 22d136e5fd..f99811acd9 100644 --- a/src/integration/blockchain/spark/spark-client.ts +++ b/src/integration/blockchain/spark/spark-client.ts @@ -1,6 +1,6 @@ import { SparkWallet } from '@buildonspark/spark-sdk'; import { Currency } from '@uniswap/sdk-core'; -import { GetConfig } from 'src/config/config'; +import { CronRole, GetConfig } from 'src/config/config'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { AsyncField } from 'src/shared/utils/async-field'; import { BlockchainTokenBalance } from '../shared/dto/blockchain-token-balance.dto'; @@ -226,10 +226,10 @@ export class SparkClient extends BlockchainClient { } private startTokenOptimization(): void { - // On-chain wallet maintenance is global work: it must run on exactly one instance. - // This timer predates the scheduler and bypasses it, so an HTTP-only instance would - // otherwise drive optimizeTokenOutputs against the same seed as the job instance. - if (!GetConfig().cronJobsEnabled) return; + // On-chain wallet maintenance is global work: it must run in exactly one process. This + // timer predates the scheduler and bypasses it, so without the role check the API process + // would drive optimizeTokenOutputs against the same seed as the worker. + if (GetConfig().cronRole === CronRole.Api) return; if (this.tokenOptimizationInterval) clearInterval(this.tokenOptimizationInterval); diff --git a/src/integration/exchange/controllers/exchange.controller.ts b/src/integration/exchange/controllers/exchange.controller.ts index e01f856443..69fc9e5ff7 100644 --- a/src/integration/exchange/controllers/exchange.controller.ts +++ b/src/integration/exchange/controllers/exchange.controller.ts @@ -20,7 +20,7 @@ import { UserActiveGuard } from 'src/shared/auth/user-active.guard'; import { UserRole } from 'src/shared/auth/user-role.enum'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { DisabledProcess, Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { Price } from '../../../subdomains/supporting/pricing/domain/entities/price'; import { TradeOrder } from '../dto/trade-order.dto'; @@ -171,7 +171,7 @@ export class ExchangeController { } // --- JOBS --- // - @DfxCron(CronExpression.EVERY_30_SECONDS, { perInstance: true, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_30_SECONDS, { scope: CronScope.Both, timeout: 1800 }) async checkTrades() { const openTrades = Object.values(this.trades).filter(({ status }) => status === TradeStatus.OPEN); for (const trade of openTrades) { diff --git a/src/jest-env.setup.ts b/src/jest-env.setup.ts index 3da86eb4f1..b2bc51f2e7 100644 --- a/src/jest-env.setup.ts +++ b/src/jest-env.setup.ts @@ -7,3 +7,9 @@ if (!process.env.REALUNIT_W2W_GAS_LOW_BALANCE_THRESHOLD) { process.env.REALUNIT_W2W_GAS_LOW_BALANCE_THRESHOLD = '0.05'; } + +// The single-process mode, matching how a test run behaves: every job is registered, none is +// filtered out by role. A spec that asserts on the role sets it explicitly and restores it. +if (!process.env.CRON_ROLE) { + process.env.CRON_ROLE = 'all'; +} diff --git a/src/shared/services/__tests__/dfx-cron.service.spec.ts b/src/shared/services/__tests__/dfx-cron.service.spec.ts index c7dab9131d..ba5a532fee 100644 --- a/src/shared/services/__tests__/dfx-cron.service.spec.ts +++ b/src/shared/services/__tests__/dfx-cron.service.spec.ts @@ -8,7 +8,7 @@ import { createMock } from '@golevelup/ts-jest'; import { DiscoveryService, MetadataScanner } from '@nestjs/core'; import { CronExpression, SchedulerRegistry } from '@nestjs/schedule'; import { ConfigService, GetConfig } from 'src/config/config'; -import { DFX_CRONJOB_PARAMS, DfxCronParams } from 'src/shared/utils/cron'; +import { CronScope, DFX_CRONJOB_PARAMS, DfxCronParams } from 'src/shared/utils/cron'; import { DfxCronService } from '../dfx-cron.service'; import { Process } from '../process.service'; @@ -44,67 +44,88 @@ function buildService(providers: { instance: object }[]): { } describe('DfxCronService', () => { - const original = process.env.CRON_JOBS_ENABLED; + const original = process.env.CRON_ROLE; const configuredJobs = [ - providerWithJob('withProcess', { expression: CronExpression.EVERY_MINUTE, process: Process.MONITOR_EVENT_LOOP }), - // A job without `process` — DISABLED_PROCESSES cannot stop this one, only the global switch can. - providerWithJob('withoutProcess', { expression: CronExpression.EVERY_MINUTE }), - // Per-instance housekeeping: must survive the switch, because it refreshes state that - // requests on this very instance read. - providerWithJob('perInstanceJob', { expression: CronExpression.EVERY_MINUTE, perInstance: true }), + providerWithJob('workerJob', { + expression: CronExpression.EVERY_MINUTE, + scope: CronScope.Worker, + process: Process.MONITOR_EVENT_LOOP, + }), + // A worker job without `process` — DISABLED_PROCESSES cannot stop this one, only the role can. + providerWithJob('workerJobWithoutProcess', { expression: CronExpression.EVERY_MINUTE, scope: CronScope.Worker }), + providerWithJob('apiJob', { expression: CronExpression.EVERY_MINUTE, scope: CronScope.Api }), + providerWithJob('bothJob', { expression: CronExpression.EVERY_MINUTE, scope: CronScope.Both }), ]; function registeredJobNames(scheduler: SchedulerRegistry): string[] { return (scheduler.addCronJob as jest.Mock).mock.calls.map(([name]) => name as string); } + function runWithRole(role: string): SchedulerRegistry { + process.env.CRON_ROLE = role; + new ConfigService(GetConfig()); + + const { service, scheduler } = buildService(configuredJobs); + service.onModuleInit(); + + return scheduler; + } + afterEach(() => { jest.clearAllMocks(); - if (original == null) delete process.env.CRON_JOBS_ENABLED; - else process.env.CRON_JOBS_ENABLED = original; + if (original == null) delete process.env.CRON_ROLE; + else process.env.CRON_ROLE = original; new ConfigService(GetConfig()); }); - it('registers every job when cron is enabled', () => { - process.env.CRON_JOBS_ENABLED = 'true'; - new ConfigService(GetConfig()); + it('registers every job in the single-process role', () => { + // The mode of local development, the test suite and any deployment without a worker: no job + // may be dropped, otherwise `all` would not reproduce today's behaviour. + const scheduler = runWithRole('all'); + + expect(registeredJobNames(scheduler)).toEqual([ + 'Object::workerJob', + 'Object::workerJobWithoutProcess', + 'Object::apiJob', + 'Object::bothJob', + ]); + expect(mockStart).toHaveBeenCalledTimes(4); + }); - const { service, scheduler } = buildService(configuredJobs); - service.onModuleInit(); + it('drops worker jobs in the API role, including those without a process', () => { + // The safety property of the API process: were a worker job still registered here, it would + // run in both processes simultaneously. Cron locks are per-process, so duplicate execution + // would go unnoticed — and DISABLED_PROCESSES cannot catch a job without a `process`. + const scheduler = runWithRole('api'); - expect(scheduler.addCronJob).toHaveBeenCalledTimes(3); - expect(mockStart).toHaveBeenCalledTimes(3); + expect(registeredJobNames(scheduler)).toEqual(['Object::apiJob', 'Object::bothJob']); }); - it('drops global jobs when cron is disabled, including those without a process', () => { - // The safety property of the HTTP-only instance: were a job without `process` still - // registered here, it would run on both the HTTP and the job instance simultaneously. - // Cron locks are per-process, so duplicate execution would go unnoticed. - process.env.CRON_JOBS_ENABLED = 'false'; - new ConfigService(GetConfig()); + it('drops API jobs in the worker role', () => { + // The counterpart: an api-scoped job drives work bound to the process holding the open + // connections, so running it in the worker would do the work where nobody can see it. + const scheduler = runWithRole('worker'); - const { service, scheduler } = buildService(configuredJobs); - service.onModuleInit(); - - expect(registeredJobNames(scheduler)).not.toContain('Object::withProcess'); - expect(registeredJobNames(scheduler)).not.toContain('Object::withoutProcess'); + expect(registeredJobNames(scheduler)).toEqual([ + 'Object::workerJob', + 'Object::workerJobWithoutProcess', + 'Object::bothJob', + ]); }); - it('keeps per-instance housekeeping when cron is disabled', () => { - // The counterpart safety property: jobs marked perInstance refresh process-local state - // (JWT denylists, the disabled-process map, local caches) that requests on THIS instance - // read. Dropping them here would freeze that state at boot — a revoked token would keep - // working on the HTTP instance until the next restart. - process.env.CRON_JOBS_ENABLED = 'false'; - new ConfigService(GetConfig()); + it('keeps jobs scoped both in every role', () => { + // Jobs scoped `both` refresh process-local state (the JWT denylists, the disabled-process + // map, local caches) that requests on THIS process read. Dropping them in either role would + // freeze that state at boot — a revoked token would keep working until the next restart. + for (const role of ['all', 'api', 'worker']) { + const scheduler = runWithRole(role); - const { service, scheduler } = buildService(configuredJobs); - service.onModuleInit(); + expect(registeredJobNames(scheduler)).toContain('Object::bothJob'); - expect(registeredJobNames(scheduler)).toEqual(['Object::perInstanceJob']); - expect(mockStart).toHaveBeenCalledTimes(1); + jest.clearAllMocks(); + } }); }); diff --git a/src/shared/services/dfx-cron.service.ts b/src/shared/services/dfx-cron.service.ts index fd717335e9..18d824b33a 100644 --- a/src/shared/services/dfx-cron.service.ts +++ b/src/shared/services/dfx-cron.service.ts @@ -2,9 +2,9 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { DiscoveryService, MetadataScanner } from '@nestjs/core'; import { CronExpression, SchedulerRegistry } from '@nestjs/schedule'; import { CronJob } from 'cron'; -import { Config } from 'src/config/config'; +import { Config, CronRole } from 'src/config/config'; import { DisabledProcess } from 'src/shared/services/process.service'; -import { DFX_CRONJOB_PARAMS, DfxCronExpression, DfxCronParams } from 'src/shared/utils/cron'; +import { CronScope, DFX_CRONJOB_PARAMS, DfxCronExpression, DfxCronParams } from 'src/shared/utils/cron'; import { LockClass } from 'src/shared/utils/lock'; import { Util } from 'src/shared/utils/util'; import { CustomCronExpression } from '../utils/custom-cron-expression'; @@ -28,6 +28,7 @@ export class DfxCronService implements OnModuleInit { ) {} onModuleInit() { + const registered: CronScope[] = []; let skipped = 0; this.discovery @@ -49,23 +50,44 @@ export class DfxCronService implements OnModuleInit { }) .filter((data) => data.params) .forEach((data) => { - // On an HTTP-only instance the global jobs belong to the job instance, but - // per-instance housekeeping must still run here: it refreshes process-local state - // such as the JWT denylists and the disabled-process map, which HTTP requests read - // on this very instance. Skipping those would freeze them at boot. - if (!Config.cronJobsEnabled && !data.params.perInstance) { + const scope = data.params.scope ?? CronScope.Both; + + if (!this.runsInThisRole(scope)) { skipped++; return; } + registered.push(scope); this.addCronJob(data); }); }); - if (skipped) { - this.logger.info( - `Cron jobs disabled on this instance (CRON_JOBS_ENABLED=false), skipped ${skipped} global job(s)`, - ); + // The effective split is read from this line, not inferred from a table: a job registered + // through a dynamically resolved provider or an abstract base class is counted here and + // nowhere else. It stays on `info` so it is findable without changing the log level. + const total = registered.length + skipped; + const byScope = Object.values(CronScope) + .map((scope) => `${scope}: ${registered.filter((s) => s === scope).length}`) + .join(', '); + + this.logger.info(`CronRole ${Config.cronRole}: registered ${registered.length} of ${total} jobs (${byScope})`); + } + + /** + * `all` runs everything, which is the single-process mode. The other two roles each run their + * own scope plus `both`, so a job maintaining process-local state that requests read is + * registered in every process. + */ + private runsInThisRole(scope: CronScope): boolean { + switch (Config.cronRole) { + case CronRole.All: + return true; + + case CronRole.Api: + return scope === CronScope.Api || scope === CronScope.Both; + + case CronRole.Worker: + return scope === CronScope.Worker || scope === CronScope.Both; } } @@ -78,6 +100,8 @@ export class DfxCronService implements OnModuleInit { this.schedulerRegisty.addCronJob(cronJobName, cronJob); cronJob.start(); + + this.logger.verbose(`Registered ${cronJobName} (${data.params.scope ?? CronScope.Both})`); } private wrapFunction(data: CronJobData) { diff --git a/src/shared/services/process.service.ts b/src/shared/services/process.service.ts index 44defcb4ac..e3caca5bae 100644 --- a/src/shared/services/process.service.ts +++ b/src/shared/services/process.service.ts @@ -3,7 +3,7 @@ import { CronExpression } from '@nestjs/schedule'; import { Config } from 'src/config/config'; import { SetStaffKycClearance } from 'src/shared/auth/staff-kyc-clearance'; import { SettingService } from '../models/setting/setting.service'; -import { DfxCron } from '../utils/cron'; +import { CronScope, DfxCron } from '../utils/cron'; export enum Process { PAY_OUT = 'PayOut', @@ -180,7 +180,7 @@ export class ProcessService implements OnModuleInit { await this.resyncStaffKycClearance(); } - @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800, perInstance: true }) + @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800, scope: CronScope.Both }) async resyncDisabledProcesses(): Promise { const allDisabledProcesses = [ ...(await this.settingService.getDisabledProcesses()), @@ -190,13 +190,13 @@ export class ProcessService implements OnModuleInit { DisabledProcesses = this.listToMap(allDisabledProcesses); } - @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800, perInstance: true }) + @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800, scope: CronScope.Both }) async resyncDeniedJwtAddresses(): Promise { const list = await this.settingService.getDeniedJwtAddresses(); DeniedJwtAddresses = new Set(list.map((a) => a.toLowerCase())); } - @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800, perInstance: true }) + @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800, scope: CronScope.Both }) async resyncDeniedJwtAccounts(): Promise { const list = await this.settingService.getDeniedJwtAccounts(); DeniedJwtAccounts = new Set(list); diff --git a/src/shared/utils/cron.ts b/src/shared/utils/cron.ts index e5afb6565a..d582a563d3 100644 --- a/src/shared/utils/cron.ts +++ b/src/shared/utils/cron.ts @@ -1,42 +1,62 @@ -import { CronExpression } from '@nestjs/schedule'; -import { Process } from '../services/process.service'; -import { CustomCronExpression } from './custom-cron-expression'; - -export interface DfxCronOptParams { - process?: Process; - useDelay?: boolean; - timeout?: number; - /** - * Marks a job as per-instance housekeeping that must run in every process, even where the - * scheduler is otherwise switched off (CRON_JOBS_ENABLED=false). - * - * Use it only for work whose effect is confined to the current process: refreshing an - * in-memory copy of global state, or expiring a local cache. Running it twice must be - * harmless by construction, because it will run on every instance. - * - * Anything that writes to the database or drives business forward is NOT per-instance — such - * a job would then execute once per instance, and cron locks are per-process and cannot - * prevent that. - */ - perInstance?: boolean; -} - -export type DfxCronExpression = CronExpression | CustomCronExpression; - -export interface DfxCronParams extends DfxCronOptParams { - expression: DfxCronExpression; -} - -export const DFX_CRONJOB_PARAMS = 'DFXCronjobParams'; - -export function DfxCron(expression: DfxCronExpression, optional?: DfxCronOptParams) { - return function (target: any, propertyKey: string, descriptor: PropertyDescriptor) { - const methodRef = target[propertyKey]; - - const params: DfxCronParams = { expression, ...optional }; - - Reflect.defineMetadata(DFX_CRONJOB_PARAMS, params, methodRef); - - return descriptor; - }; -} +import { CronExpression } from '@nestjs/schedule'; +import { Process } from '../services/process.service'; +import { CustomCronExpression } from './custom-cron-expression'; + +/** + * Which process a job belongs to. + * + * The distinction is not about importance but about where a job's effect is visible. Most jobs + * only touch the database or an external system, so any process can run them and exactly one + * should. The exceptions are jobs maintaining state that lives inside the process itself - a + * class field, a module-global variable, a Node process metric - because such state is only + * useful in the process whose requests read it. + */ +export enum CronScope { + /** Worker process only. The normal case: anything writing to the database or driving business forward. */ + Worker = 'worker', + /** + * API process only. Maintains or measures state read exclusively from a request path, or + * drives work bound to the connections that process holds open. + */ + Api = 'api', + /** + * Every process. Maintains or measures process-local state that both sides read. + * + * Running such a job twice must be harmless by construction: refreshing an in-memory copy of + * global state, expiring a local cache or writing a log line qualifies. Writing to the + * database or driving business forward does not - cron locks are per-process and cannot + * prevent duplicate execution across processes. + */ + Both = 'both', +} + +export interface DfxCronOptParams { + process?: Process; + useDelay?: boolean; + timeout?: number; + /** + * Which process runs this job. A job without a scope runs in every process, which is what + * every job did before the scope existed. + */ + scope?: CronScope; +} + +export type DfxCronExpression = CronExpression | CustomCronExpression; + +export interface DfxCronParams extends DfxCronOptParams { + expression: DfxCronExpression; +} + +export const DFX_CRONJOB_PARAMS = 'DFXCronjobParams'; + +export function DfxCron(expression: DfxCronExpression, optional?: DfxCronOptParams) { + return function (target: any, propertyKey: string, descriptor: PropertyDescriptor) { + const methodRef = target[propertyKey]; + + const params: DfxCronParams = { expression, ...optional }; + + Reflect.defineMetadata(DFX_CRONJOB_PARAMS, params, methodRef); + + return descriptor; + }; +} diff --git a/src/subdomains/core/history/controllers/transaction.controller.ts b/src/subdomains/core/history/controllers/transaction.controller.ts index f158b7b9a4..5ccb96d8a8 100644 --- a/src/subdomains/core/history/controllers/transaction.controller.ts +++ b/src/subdomains/core/history/controllers/transaction.controller.ts @@ -31,7 +31,7 @@ import { UserRole } from 'src/shared/auth/user-role.enum'; import { isFiatDto } from 'src/shared/models/active'; import { AssetDtoMapper } from 'src/shared/models/asset/dto/asset-dto.mapper'; import { FiatService } from 'src/shared/models/fiat/fiat.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { AmountType, Util } from 'src/shared/utils/util'; import { BankDataService } from 'src/subdomains/generic/user/models/bank-data/bank-data.service'; import { UserData } from 'src/subdomains/generic/user/models/user-data/user-data.entity'; @@ -116,7 +116,7 @@ export class TransactionController { ) {} // --- JOBS --- // - @DfxCron(CronExpression.EVERY_MINUTE, { perInstance: true }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Both }) checkLists() { for (const [key, refundData] of this.refundList.entries()) { if (!this.isRefundDataValid(refundData)) this.refundList.delete(key); diff --git a/src/subdomains/core/monitoring/monitor-connection-pool.service.ts b/src/subdomains/core/monitoring/monitor-connection-pool.service.ts index b5215ec590..338474a892 100644 --- a/src/subdomains/core/monitoring/monitor-connection-pool.service.ts +++ b/src/subdomains/core/monitoring/monitor-connection-pool.service.ts @@ -3,7 +3,7 @@ import { CronExpression } from '@nestjs/schedule'; import { Config } from 'src/config/config'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { DataSource } from 'typeorm'; import { PostgresConnectionOptions } from 'typeorm/driver/postgres/PostgresConnectionOptions'; import { PostgresDriver } from 'typeorm/driver/postgres/PostgresDriver'; @@ -19,7 +19,7 @@ export class MonitorConnectionPoolService { this.dbConnectionPool = dbDriver.master; } - @DfxCron(CronExpression.EVERY_SECOND, { perInstance: true, process: Process.MONITOR_CONNECTION_POOL }) + @DfxCron(CronExpression.EVERY_SECOND, { scope: CronScope.Both, process: Process.MONITOR_CONNECTION_POOL }) monitorConnectionPool() { const dbOptions = Config.database as PostgresConnectionOptions; const dbMaxPoolConnections = dbOptions.poolSize ?? 10; @@ -37,7 +37,7 @@ export class MonitorConnectionPoolService { } } - @DfxCron(CronExpression.EVERY_10_SECONDS, { perInstance: true, process: Process.MONITOR_CONNECTION_POOL }) + @DfxCron(CronExpression.EVERY_10_SECONDS, { scope: CronScope.Both, process: Process.MONITOR_CONNECTION_POOL }) monitorConnectionPoolStatic() { const total = this.dbConnectionPool.totalCount; const idle = this.dbConnectionPool.idleCount; diff --git a/src/subdomains/core/monitoring/monitor-event-loop.service.ts b/src/subdomains/core/monitoring/monitor-event-loop.service.ts index efc5dc3ba3..c674213139 100644 --- a/src/subdomains/core/monitoring/monitor-event-loop.service.ts +++ b/src/subdomains/core/monitoring/monitor-event-loop.service.ts @@ -3,7 +3,7 @@ import { CronExpression } from '@nestjs/schedule'; import { monitorEventLoopDelay, performance } from 'perf_hooks'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; @Injectable() export class MonitorEventLoopService implements OnModuleDestroy { @@ -22,7 +22,7 @@ export class MonitorEventLoopService implements OnModuleDestroy { this.histogram.disable(); } - @DfxCron(CronExpression.EVERY_10_SECONDS, { perInstance: true, process: Process.MONITOR_EVENT_LOOP }) + @DfxCron(CronExpression.EVERY_10_SECONDS, { scope: CronScope.Both, process: Process.MONITOR_EVENT_LOOP }) monitorEventLoop(): void { const toMs = (ns: number) => Math.round(ns / 1e6); diff --git a/src/subdomains/core/payment-link/services/payment-link-fee.service.ts b/src/subdomains/core/payment-link/services/payment-link-fee.service.ts index 242e37070a..bab5ceee52 100644 --- a/src/subdomains/core/payment-link/services/payment-link-fee.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-fee.service.ts @@ -6,7 +6,7 @@ import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.e import { PaymentLinkBlockchains } from 'src/integration/blockchain/shared/util/blockchain.util'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { PayoutBitcoinService } from 'src/subdomains/supporting/payout/services/payout-bitcoin.service'; import { PayoutFiroService } from 'src/subdomains/supporting/payout/services/payout-firo.service'; @@ -38,7 +38,7 @@ export class PaymentLinkFeeService implements OnModuleInit { } // --- JOBS --- // - @DfxCron(CronExpression.EVERY_MINUTE, { perInstance: true, process: Process.UPDATE_BLOCKCHAIN_FEE }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Both, process: Process.UPDATE_BLOCKCHAIN_FEE }) async updateFees(): Promise { if (GetConfig().environment === Environment.LOC) return; diff --git a/src/subdomains/core/statistic/statistic.service.ts b/src/subdomains/core/statistic/statistic.service.ts index 85b1231089..52adae78a0 100644 --- a/src/subdomains/core/statistic/statistic.service.ts +++ b/src/subdomains/core/statistic/statistic.service.ts @@ -3,7 +3,7 @@ import { CronExpression } from '@nestjs/schedule'; import { Config } from 'src/config/config'; import { SettingService } from 'src/shared/models/setting/setting.service'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { SellService } from 'src/subdomains/core/sell-crypto/route/sell.service'; import { UserService } from 'src/subdomains/generic/user/models/user/user.service'; @@ -25,7 +25,7 @@ export class StatisticService implements OnModuleInit { void this.doUpdate(); } - @DfxCron(CronExpression.EVERY_HOUR, { perInstance: true, process: Process.UPDATE_STATISTIC, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Api, process: Process.UPDATE_STATISTIC, timeout: 7200 }) async doUpdate(): Promise { this.statistic = { totalVolume: { diff --git a/src/subdomains/generic/kyc/services/tfa.service.ts b/src/subdomains/generic/kyc/services/tfa.service.ts index 3f5795ea19..9b4af160a5 100644 --- a/src/subdomains/generic/kyc/services/tfa.service.ts +++ b/src/subdomains/generic/kyc/services/tfa.service.ts @@ -12,7 +12,7 @@ import { CronExpression } from '@nestjs/schedule'; import { generateSecret, verifyToken } from 'node-2fa'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { TfaLogRepository } from 'src/subdomains/generic/kyc/repositories/tfa-log.repository'; import { MailContext, MailType } from 'src/subdomains/supporting/notification/enums'; @@ -50,7 +50,7 @@ export class TfaService { private readonly notificationService: NotificationService, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { perInstance: true, process: Process.TFA_CACHE }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Both, process: Process.TFA_CACHE }) processCleanupSecretCache() { const now = new Date(); diff --git a/src/subdomains/generic/user/models/auth/auth-lnurl.service.ts b/src/subdomains/generic/user/models/auth/auth-lnurl.service.ts index d0fbfdf8dd..f4a486bc22 100644 --- a/src/subdomains/generic/user/models/auth/auth-lnurl.service.ts +++ b/src/subdomains/generic/user/models/auth/auth-lnurl.service.ts @@ -6,7 +6,7 @@ import { Config } from 'src/config/config'; import { LightningHelper } from 'src/integration/lightning/lightning-helper'; import { IpLogService } from 'src/shared/models/ip-log/ip-log.service'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { AuthService } from 'src/subdomains/generic/user/models/auth/auth.service'; import { @@ -36,7 +36,7 @@ export class AuthLnUrlService { private readonly ipLogService: IpLogService, ) {} - @DfxCron(CronExpression.EVERY_30_SECONDS, { perInstance: true, process: Process.LNURL_AUTH_CACHE }) + @DfxCron(CronExpression.EVERY_30_SECONDS, { scope: CronScope.Both, process: Process.LNURL_AUTH_CACHE }) processCleanupAccessToken() { const before30SecTime = Util.secondsBefore(30).getTime(); @@ -47,7 +47,7 @@ export class AuthLnUrlService { keysToBeDeleted.forEach((k) => this.authCache.delete(k)); } - @DfxCron(CronExpression.EVERY_5_MINUTES, { perInstance: true, process: Process.LNURL_AUTH_CACHE }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.Both, process: Process.LNURL_AUTH_CACHE }) processCleanupAuthCache() { const before5MinTime = Util.minutesBefore(5).getTime(); diff --git a/src/subdomains/generic/user/models/auth/auth.service.ts b/src/subdomains/generic/user/models/auth/auth.service.ts index ab5e664440..f77c22e9d8 100644 --- a/src/subdomains/generic/user/models/auth/auth.service.ts +++ b/src/subdomains/generic/user/models/auth/auth.service.ts @@ -22,7 +22,7 @@ import { LanguageService } from 'src/shared/models/language/language.service'; import { SettingService } from 'src/shared/models/setting/setting.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { AsyncCache, CacheItemResetPeriod } from 'src/shared/utils/async-cache'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { RefService } from 'src/subdomains/core/referral/process/ref.service'; import { KycStepName } from 'src/subdomains/generic/kyc/enums/kyc-step-name.enum'; @@ -98,7 +98,7 @@ export class AuthService { @Inject(forwardRef(() => KycService)) private readonly kycService: KycService, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { perInstance: true }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Both }) checkLists() { for (const [key, challenge] of this.challengeList.entries()) { if (!this.isChallengeValid(challenge)) { diff --git a/src/subdomains/generic/user/models/user-data/user-data.service.ts b/src/subdomains/generic/user/models/user-data/user-data.service.ts index f619220797..6cc4a9e281 100644 --- a/src/subdomains/generic/user/models/user-data/user-data.service.ts +++ b/src/subdomains/generic/user/models/user-data/user-data.service.ts @@ -23,7 +23,7 @@ import { SettingService } from 'src/shared/models/setting/setting.service'; import { RepositoryFactory } from 'src/shared/repositories/repository.factory'; import { ApiKeyService } from 'src/shared/services/api-key.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { AmountType, Util } from 'src/shared/utils/util'; import { CheckStatus } from 'src/subdomains/core/aml/enums/check-status.enum'; import { CustodyService } from 'src/subdomains/core/custody/services/custody.service'; @@ -833,7 +833,7 @@ export class UserDataService { return this.doUpdateUserMail(userData, cacheEntry.mail); } - @DfxCron(CronExpression.EVERY_MINUTE, { perInstance: true }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Both }) processCleanupMailSecretCache(): void { const now = new Date(); diff --git a/src/subdomains/supporting/payment/services/transaction-helper.ts b/src/subdomains/supporting/payment/services/transaction-helper.ts index 2411db46bb..42f1ab4e8e 100644 --- a/src/subdomains/supporting/payment/services/transaction-helper.ts +++ b/src/subdomains/supporting/payment/services/transaction-helper.ts @@ -17,7 +17,7 @@ import { FiatService } from 'src/shared/models/fiat/fiat.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { DisabledProcess, Process } from 'src/shared/services/process.service'; import { AsyncCache, CacheItemResetPeriod } from 'src/shared/utils/async-cache'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { AmountType, Util } from 'src/shared/utils/util'; import { AmlRule } from 'src/subdomains/core/aml/enums/aml-rule.enum'; import { AmlHelperService } from 'src/subdomains/core/aml/services/aml-helper.service'; @@ -87,7 +87,7 @@ export class TransactionHelper implements OnModuleInit { void this.updateCache(); } - @DfxCron(CronExpression.EVERY_5_MINUTES, { perInstance: true }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.Both }) async updateCache() { this.transactionSpecifications = await this.specRepo.find(); } From e80ac418c322cd4da36e92b5ff7e8e1a245b1a90 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:22 +0200 Subject: [PATCH 11/86] Require a scope on every cron job The role only decides which jobs a process registers; it cannot decide which process a job belongs to. That answer has to come from the job, and leaving it optional puts the silent failure on the wrong side: a job that forgets the field keeps running everywhere, and one wrongly scoped `worker` stops refreshing the cache the request path reads, in both cases without an error. The alternative - a default plus a list of exceptions - moves the decision into a hand-maintained list the compiler never sees. Such a list grows, goes stale, and pinning it in a test proves the state of the list rather than the property it stands for. Making the field mandatory moves the check into the compiler, where it stays complete for every job added from here on. The price is paid once, here: 116 decorators gain `scope: CronScope.Worker`, which is what they already did and what any job touching only the database or an external system should do. The seventeen exceptions were classified in the previous commit. The change is mechanical and carries no behaviour: with CRON_ROLE=all every job is registered regardless of its scope. --- .../blockchain/deuro/deuro.service.ts | 4 +- .../frankencoin/frankencoin.service.ts | 4 +- .../blockchain/juice/juice.service.ts | 4 +- .../shared/evm/evm-decimals.service.ts | 4 +- .../blockchain-config-check.service.ts | 4 +- .../blockchain/zano/services/zano.service.ts | 4 +- .../exchange/services/exchange-tx.service.ts | 8 ++- src/shared/services/dfx-cron.service.ts | 8 +-- src/shared/utils/cron.ts | 25 +++++--- .../services/ledger-booking-job.service.ts | 62 +++++++++++++++---- .../services/ledger-cutover.service.ts | 4 +- .../services/ledger-mark-to-market.service.ts | 4 +- .../services/ledger-reconciliation.service.ts | 4 +- .../core/aml/services/sanction.service.ts | 4 +- .../services/buy-crypto-job.service.ts | 10 ++- .../core/buy-crypto/routes/buy/buy.service.ts | 6 +- .../buy-crypto/routes/swap/swap.service.ts | 6 +- .../custody/services/custody-job.service.ts | 6 +- .../services/faucet-request.service.ts | 4 +- .../liquidity-management-pipeline.service.ts | 8 ++- .../liquidity-management-rule.service.ts | 8 ++- .../services/liquidity-management.service.ts | 8 ++- .../core/monitoring/observers/aml.observer.ts | 4 +- .../monitoring/observers/bank.observer.ts | 4 +- .../monitoring/observers/checkout.observer.ts | 4 +- .../monitoring/observers/exchange.observer.ts | 4 +- .../observers/external-services.observer.ts | 4 +- .../observers/liquidity.observer.ts | 4 +- .../observers/node-balance.observer.ts | 4 +- .../observers/node-health.observer.ts | 4 +- .../monitoring/observers/payment.observer.ts | 4 +- .../observers/realunit-w2w-gas.observer.ts | 4 +- .../monitoring/observers/user.observer.ts | 4 +- .../services/payment-cron.service.ts | 8 +-- .../core/referral/process/ref.service.ts | 4 +- .../reward/services/ref-reward-job.service.ts | 6 +- .../process/services/buy-fiat-job.service.ts | 6 +- .../services/buy-fiat-notification.service.ts | 4 +- .../core/sell-crypto/route/sell.service.ts | 6 +- .../trading/services/trading-job.service.ts | 8 +-- src/subdomains/generic/admin/admin.service.ts | 4 +- .../kyc/services/kyc-notification.service.ts | 4 +- .../generic/kyc/services/kyc.service.ts | 6 +- .../models/bank-data/bank-data.service.ts | 8 ++- .../organization/organization.service.ts | 4 +- .../user-data/jwt-revocation-sync.service.ts | 4 +- .../models/user-data/user-data-job.service.ts | 4 +- .../user-data-notification.service.ts | 4 +- .../models/user-data/user-data.service.ts | 4 +- .../user/models/user/user-job.service.ts | 4 +- .../generic/user/models/user/user.service.ts | 6 +- .../webhook/webhook-notification.service.ts | 4 +- .../bank-tx-return-notification.service.ts | 8 ++- .../bank-tx-return/bank-tx-return.service.ts | 4 +- .../bank-tx/services/bank-tx.service.ts | 6 +- .../bank/bank-account/bank-account.service.ts | 8 +-- ...n-frick-issuance-reconciliation.service.ts | 3 +- .../supporting/dex/services/dex.service.ts | 4 +- .../fiat-output/fiat-output-frick.service.ts | 4 +- .../fiat-output/fiat-output-job.service.ts | 8 +-- .../services/fiat-payin-sync.service.ts | 4 +- .../supporting/log/log-job.service.ts | 4 +- src/subdomains/supporting/log/log.service.ts | 4 +- .../services/notification-job.service.ts | 4 +- .../services/payin-notification.service.ts | 4 +- .../payin/services/payin.service.ts | 10 +-- .../register/impl/base/citrea.strategy.ts | 4 +- .../register/impl/bitcoin.strategy.ts | 4 +- .../register/impl/cardano.strategy.ts | 4 +- .../strategies/register/impl/firo.strategy.ts | 4 +- .../strategies/register/impl/icp.strategy.ts | 4 +- .../register/impl/monero.strategy.ts | 4 +- .../strategies/register/impl/zano.strategy.ts | 4 +- .../payment/services/fee.service.ts | 8 ++- .../transaction-notification.service.ts | 4 +- .../services/transaction-request.service.ts | 10 ++- .../payout/services/payout.service.ts | 4 +- .../services/asset-prices-job.service.ts | 6 +- .../pricing/services/fiat-prices.service.ts | 4 +- .../realunit/realunit-job.service.ts | 14 ++++- .../limit-request-notification.service.ts | 8 ++- .../services/support-escalation.service.ts | 4 +- .../services/support-issue-job.service.ts | 6 +- 83 files changed, 304 insertions(+), 208 deletions(-) diff --git a/src/integration/blockchain/deuro/deuro.service.ts b/src/integration/blockchain/deuro/deuro.service.ts index 002ca71236..c815cabb6a 100644 --- a/src/integration/blockchain/deuro/deuro.service.ts +++ b/src/integration/blockchain/deuro/deuro.service.ts @@ -5,7 +5,7 @@ import { Contract } from 'ethers'; import { Config } from 'src/config/config'; import { Asset } from 'src/shared/models/asset/asset.entity'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { CreateLogDto } from 'src/subdomains/supporting/log/dto/create-log.dto'; import { LogSeverity } from 'src/subdomains/supporting/log/log.entity'; @@ -60,7 +60,7 @@ export class DEuroService extends FrankencoinBasedService implements OnModuleIni this.deuroClient = new DEuroClient(this.getEvmClient()); } - @DfxCron(CronExpression.EVERY_10_MINUTES, { process: Process.DEURO_LOG_INFO }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.DEURO_LOG_INFO }) async processLogInfo(): Promise { if (!Config.blockchain.deuro.graphUrl || !Config.blockchain.deuro.apiUrl) { this.logger.warn('DEuro graphUrl/apiUrl not configured - skipping processLogInfo'); diff --git a/src/integration/blockchain/frankencoin/frankencoin.service.ts b/src/integration/blockchain/frankencoin/frankencoin.service.ts index 468dd42899..5f12785751 100644 --- a/src/integration/blockchain/frankencoin/frankencoin.service.ts +++ b/src/integration/blockchain/frankencoin/frankencoin.service.ts @@ -4,7 +4,7 @@ import { CronExpression } from '@nestjs/schedule'; import { Contract } from 'ethers'; import { Config } from 'src/config/config'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { CreateLogDto } from 'src/subdomains/supporting/log/dto/create-log.dto'; import { LogSeverity } from 'src/subdomains/supporting/log/log.entity'; import { LogService } from 'src/subdomains/supporting/log/log.service'; @@ -51,7 +51,7 @@ export class FrankencoinService extends FrankencoinBasedService implements OnMod this.frankencoinClient = new FrankencoinClient(this.getEvmClient()); } - @DfxCron(CronExpression.EVERY_10_MINUTES, { process: Process.FRANKENCOIN_LOG_INFO }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.FRANKENCOIN_LOG_INFO }) async processLogInfo() { if (!Config.blockchain.frankencoin.contractAddress.xchf) { this.logger.warn('Frankencoin xchf contract not configured - skipping processLogInfo'); diff --git a/src/integration/blockchain/juice/juice.service.ts b/src/integration/blockchain/juice/juice.service.ts index 0b6e499db5..660fe114c9 100644 --- a/src/integration/blockchain/juice/juice.service.ts +++ b/src/integration/blockchain/juice/juice.service.ts @@ -5,7 +5,7 @@ import { Contract } from 'ethers'; import { Config } from 'src/config/config'; import { Asset } from 'src/shared/models/asset/asset.entity'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { CreateLogDto } from 'src/subdomains/supporting/log/dto/create-log.dto'; import { LogSeverity } from 'src/subdomains/supporting/log/log.entity'; @@ -61,7 +61,7 @@ export class JuiceService extends FrankencoinBasedService implements OnModuleIni return this.registryService.getClient(Blockchain.CITREA) as EvmClient; } - @DfxCron(CronExpression.EVERY_10_MINUTES, { process: Process.JUICE_LOG_INFO }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.JUICE_LOG_INFO }) async processLogInfo(): Promise { if (!Config.blockchain.juice.graphUrl || !Config.blockchain.juice.apiUrl) { this.logger.warn('Juice graphUrl/apiUrl not configured - skipping processLogInfo'); diff --git a/src/integration/blockchain/shared/evm/evm-decimals.service.ts b/src/integration/blockchain/shared/evm/evm-decimals.service.ts index 19ac56f539..099def2332 100644 --- a/src/integration/blockchain/shared/evm/evm-decimals.service.ts +++ b/src/integration/blockchain/shared/evm/evm-decimals.service.ts @@ -5,7 +5,7 @@ import { AssetService } from 'src/shared/models/asset/asset.service'; import { UpdateResult } from 'src/shared/models/entity'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { BlockchainRegistryService } from '../services/blockchain-registry.service'; import { EvmBlockchains } from '../util/blockchain.util'; @@ -19,7 +19,7 @@ export class EvmDecimalsService { ) {} // --- JOBS --- // - @DfxCron(CronExpression.EVERY_HOUR, { process: Process.ASSET_DECIMALS, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, process: Process.ASSET_DECIMALS, timeout: 1800 }) async setDecimals() { const assets = await this.assetService.getEvmAssetsWithoutDecimals(EvmBlockchains); diff --git a/src/integration/blockchain/shared/services/blockchain-config-check.service.ts b/src/integration/blockchain/shared/services/blockchain-config-check.service.ts index 5341e72c0f..018c5e8e7c 100644 --- a/src/integration/blockchain/shared/services/blockchain-config-check.service.ts +++ b/src/integration/blockchain/shared/services/blockchain-config-check.service.ts @@ -7,7 +7,7 @@ import { BlockchainRegistryService } from 'src/integration/blockchain/shared/ser import { TestBlockchains } from 'src/integration/blockchain/shared/util/blockchain.util'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; @Injectable() export class BlockchainConfigCheckService { @@ -20,7 +20,7 @@ export class BlockchainConfigCheckService { // reports what a client can actually tell us today: a missing Tatum API key (Cardano, Solana, Tron) and a // missing node URL (Bitcoin, Firo). Clients that build unconditionally report configured, so silence here // is not a full-coverage statement - @DfxCron(CronExpression.EVERY_5_MINUTES, { process: Process.BLOCKCHAIN_CONFIG_CHECK }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.Worker, process: Process.BLOCKCHAIN_CONFIG_CHECK }) logUnconfiguredClients(): void { if (Config.environment !== Environment.PRD) return; diff --git a/src/integration/blockchain/zano/services/zano.service.ts b/src/integration/blockchain/zano/services/zano.service.ts index b9d2cfc61b..508c34e59c 100644 --- a/src/integration/blockchain/zano/services/zano.service.ts +++ b/src/integration/blockchain/zano/services/zano.service.ts @@ -5,7 +5,7 @@ import { Asset } from 'src/shared/models/asset/asset.entity'; import { AssetService } from 'src/shared/models/asset/asset.service'; import { HttpService } from 'src/shared/services/http.service'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { Deposit } from 'src/subdomains/supporting/address-pool/deposit/deposit.entity'; import { DepositService } from 'src/subdomains/supporting/address-pool/deposit/deposit.service'; @@ -40,7 +40,7 @@ export class ZanoService extends BlockchainService implements OnModuleInit { } // --- JOBS --- // - @DfxCron(CronExpression.EVERY_10_MINUTES, { process: Process.ZANO_ASSET_WHITELIST }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.ZANO_ASSET_WHITELIST }) async setupAssetWhitelist(): Promise { if (await this.isHealthy()) { const zanoTokens = await this.assetService.getTokens(Blockchain.ZANO); diff --git a/src/integration/exchange/services/exchange-tx.service.ts b/src/integration/exchange/services/exchange-tx.service.ts index c3cb9d82bd..41152b50f0 100644 --- a/src/integration/exchange/services/exchange-tx.service.ts +++ b/src/integration/exchange/services/exchange-tx.service.ts @@ -5,7 +5,7 @@ import { AssetService } from 'src/shared/models/asset/asset.service'; import { FiatService } from 'src/shared/models/fiat/fiat.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { DisabledProcess, Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { PriceCurrency, @@ -96,7 +96,11 @@ export class ExchangeTxService implements OnModuleInit { //*** JOBS ***// - @DfxCron(CronExpression.EVERY_5_MINUTES, { process: Process.EXCHANGE_TX_SYNC, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { + scope: CronScope.Worker, + process: Process.EXCHANGE_TX_SYNC, + timeout: 1800, + }) async syncExchangeJob() { await this.syncExchanges(); } diff --git a/src/shared/services/dfx-cron.service.ts b/src/shared/services/dfx-cron.service.ts index 18d824b33a..2a52ff6920 100644 --- a/src/shared/services/dfx-cron.service.ts +++ b/src/shared/services/dfx-cron.service.ts @@ -50,14 +50,12 @@ export class DfxCronService implements OnModuleInit { }) .filter((data) => data.params) .forEach((data) => { - const scope = data.params.scope ?? CronScope.Both; - - if (!this.runsInThisRole(scope)) { + if (!this.runsInThisRole(data.params.scope)) { skipped++; return; } - registered.push(scope); + registered.push(data.params.scope); this.addCronJob(data); }); }); @@ -101,7 +99,7 @@ export class DfxCronService implements OnModuleInit { this.schedulerRegisty.addCronJob(cronJobName, cronJob); cronJob.start(); - this.logger.verbose(`Registered ${cronJobName} (${data.params.scope ?? CronScope.Both})`); + this.logger.verbose(`Registered ${cronJobName} (${data.params.scope})`); } private wrapFunction(data: CronJobData) { diff --git a/src/shared/utils/cron.ts b/src/shared/utils/cron.ts index d582a563d3..8e610d2b99 100644 --- a/src/shared/utils/cron.ts +++ b/src/shared/utils/cron.ts @@ -34,26 +34,35 @@ export interface DfxCronOptParams { process?: Process; useDelay?: boolean; timeout?: number; - /** - * Which process runs this job. A job without a scope runs in every process, which is what - * every job did before the scope existed. - */ - scope?: CronScope; +} + +/** + * Parameters of a cron job. `scope` is mandatory and has no default. + * + * A wrong classification fails silently - a job wrongly scoped `worker` leaves the cache it + * maintains empty in the process that reads it, with no error anywhere. A default plus a list + * of exceptions moves that decision into a hand-maintained list the compiler never sees, and + * such a list grows and goes stale; pinning it in a test proves the state of the list, not the + * property it stands for. Requiring the field puts the question in front of whoever adds a job, + * which is the only check that stays complete as jobs are added. + */ +export interface DfxCronRequiredParams extends DfxCronOptParams { + scope: CronScope; } export type DfxCronExpression = CronExpression | CustomCronExpression; -export interface DfxCronParams extends DfxCronOptParams { +export interface DfxCronParams extends DfxCronRequiredParams { expression: DfxCronExpression; } export const DFX_CRONJOB_PARAMS = 'DFXCronjobParams'; -export function DfxCron(expression: DfxCronExpression, optional?: DfxCronOptParams) { +export function DfxCron(expression: DfxCronExpression, required: DfxCronRequiredParams) { return function (target: any, propertyKey: string, descriptor: PropertyDescriptor) { const methodRef = target[propertyKey]; - const params: DfxCronParams = { expression, ...optional }; + const params: DfxCronParams = { expression, ...required }; Reflect.defineMetadata(DFX_CRONJOB_PARAMS, params, methodRef); diff --git a/src/subdomains/core/accounting/services/ledger-booking-job.service.ts b/src/subdomains/core/accounting/services/ledger-booking-job.service.ts index 0675a8e3a7..671c393a89 100644 --- a/src/subdomains/core/accounting/services/ledger-booking-job.service.ts +++ b/src/subdomains/core/accounting/services/ledger-booking-job.service.ts @@ -3,7 +3,7 @@ import { CronExpression } from '@nestjs/schedule'; import { Config } from 'src/config/config'; import { SettingService } from 'src/shared/models/setting/setting.service'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { BankTxConsumer } from './consumers/bank-tx.consumer'; import { BuyCryptoConsumer } from './consumers/buy-crypto.consumer'; import { BuyFiatConsumer } from './consumers/buy-fiat.consumer'; @@ -51,59 +51,95 @@ export class LedgerBookingJobService { return (await this.settingService.get(CUTOVER_LOG_ID_KEY)) != null; } - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.LEDGER_BOOKING_BANK_TX, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { + scope: CronScope.Worker, + process: Process.LEDGER_BOOKING_BANK_TX, + timeout: 1800, + }) async runBankTx(): Promise { if (!(await this.isLedgerReady())) return; await this.bankTxConsumer.process(); } // ExchangeTx + ExchangeTrade are ONE @DfxCron method → one flag (Minor R8-1): deposit/withdrawal then trade - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.LEDGER_BOOKING_EXCHANGE_TX, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { + scope: CronScope.Worker, + process: Process.LEDGER_BOOKING_EXCHANGE_TX, + timeout: 1800, + }) async runExchangeTx(): Promise { if (!(await this.isLedgerReady())) return; await this.exchangeTxConsumer.process(); } - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.LEDGER_BOOKING_CRYPTO_INPUT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { + scope: CronScope.Worker, + process: Process.LEDGER_BOOKING_CRYPTO_INPUT, + timeout: 1800, + }) async runCryptoInput(): Promise { if (!(await this.isLedgerReady())) return; await this.cryptoInputConsumer.process(); } - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.LEDGER_BOOKING_PAYOUT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { + scope: CronScope.Worker, + process: Process.LEDGER_BOOKING_PAYOUT, + timeout: 1800, + }) async runPayoutOrder(): Promise { if (!(await this.isLedgerReady())) return; await this.payoutOrderConsumer.process(); } - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.LEDGER_BOOKING_BUY_CRYPTO, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { + scope: CronScope.Worker, + process: Process.LEDGER_BOOKING_BUY_CRYPTO, + timeout: 1800, + }) async runBuyCrypto(): Promise { if (!(await this.isLedgerReady())) return; await this.buyCryptoConsumer.process(); } - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.LEDGER_BOOKING_BUY_FIAT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { + scope: CronScope.Worker, + process: Process.LEDGER_BOOKING_BUY_FIAT, + timeout: 1800, + }) async runBuyFiat(): Promise { if (!(await this.isLedgerReady())) return; await this.buyFiatConsumer.process(); } // §4.8 — bridge-only (skips exchange/DfxDex movements booked by their authoritative consumers) - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.LEDGER_BOOKING_LIQUIDITY_MANAGEMENT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { + scope: CronScope.Worker, + process: Process.LEDGER_BOOKING_LIQUIDITY_MANAGEMENT, + timeout: 1800, + }) async runLiquidityMgmt(): Promise { if (!(await this.isLedgerReady())) return; await this.liquidityMgmtConsumer.process(); } // §4.8a — DfxDex purchase/sell on-chain swaps (own flag, Hard Constraint #5) - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.LEDGER_BOOKING_LIQUIDITY_ORDER, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { + scope: CronScope.Worker, + process: Process.LEDGER_BOOKING_LIQUIDITY_ORDER, + timeout: 1800, + }) async runLiquidityOrderDex(): Promise { if (!(await this.isLedgerReady())) return; await this.liquidityOrderDexConsumer.process(); } // §4.9 — arbitrage swaps (own flag, Hard Constraint #5) - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.LEDGER_BOOKING_TRADING_ORDER, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { + scope: CronScope.Worker, + process: Process.LEDGER_BOOKING_TRADING_ORDER, + timeout: 1800, + }) async runTradingOrder(): Promise { if (!(await this.isLedgerReady())) return; await this.tradingOrderConsumer.process(); @@ -113,7 +149,11 @@ export class LedgerBookingJobService { // Frick #4252) never get an ASSET account from the cutover-only bootstrap and wedge their consumer fail-loud // ("CoA bootstrap missing"). bootstrap() is idempotent (findOrCreate, §3), so a recurring re-run is a no-op // once complete. Pre-cutover the cutover run owns the bootstrap → gate on isLedgerReady. - @DfxCron(CronExpression.EVERY_5_MINUTES, { process: Process.LEDGER_COA_BOOTSTRAP, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { + scope: CronScope.Worker, + process: Process.LEDGER_COA_BOOTSTRAP, + timeout: 1800, + }) async runCoaBootstrap(): Promise { if (!(await this.isLedgerReady())) return; await this.bootstrapService.bootstrap(); diff --git a/src/subdomains/core/accounting/services/ledger-cutover.service.ts b/src/subdomains/core/accounting/services/ledger-cutover.service.ts index d803665a4b..6662787a5e 100644 --- a/src/subdomains/core/accounting/services/ledger-cutover.service.ts +++ b/src/subdomains/core/accounting/services/ledger-cutover.service.ts @@ -7,7 +7,7 @@ import { Asset } from 'src/shared/models/asset/asset.entity'; import { SettingService } from 'src/shared/models/setting/setting.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { BuyCrypto } from 'src/subdomains/core/buy-crypto/process/entities/buy-crypto.entity'; import { LiquidityManagementOrder } from 'src/subdomains/core/liquidity-management/entities/liquidity-management-order.entity'; @@ -103,7 +103,7 @@ export class LedgerCutoverService { * a crash never breaks the boot/cron run, leaves `ledgerCutoverLogId` unset → all consumers no-op (§4 gate). * The cron no-ops immediately once the flag is set, so it effectively runs once and is otherwise idle. */ - @DfxCron(CronExpression.EVERY_5_MINUTES, { process: Process.LEDGER_CUTOVER }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.Worker, process: Process.LEDGER_CUTOVER }) async run(): Promise { // Two deliberate checks: master switch (hard off, no DB) vs. already-cut-over (setting). Do not merge them. if (!Config.ledger.enabled) return; diff --git a/src/subdomains/core/accounting/services/ledger-mark-to-market.service.ts b/src/subdomains/core/accounting/services/ledger-mark-to-market.service.ts index 3735660251..37e2521d20 100644 --- a/src/subdomains/core/accounting/services/ledger-mark-to-market.service.ts +++ b/src/subdomains/core/accounting/services/ledger-mark-to-market.service.ts @@ -3,7 +3,7 @@ import { CronExpression } from '@nestjs/schedule'; import { Config } from 'src/config/config'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { In } from 'typeorm'; import { AccountType, LedgerAccount } from '../entities/ledger-account.entity'; @@ -45,7 +45,7 @@ export class LedgerMarkToMarketService { private readonly ledgerLegRepository: LedgerLegRepository, ) {} - @DfxCron(CronExpression.EVERY_DAY_AT_4AM, { process: Process.LEDGER_MARK_TO_MARKET }) + @DfxCron(CronExpression.EVERY_DAY_AT_4AM, { scope: CronScope.Worker, process: Process.LEDGER_MARK_TO_MARKET }) async run(): Promise { if (!(await this.jobService.isLedgerReady())) return; // cutover-gate (Blocker R1-6) applies here too diff --git a/src/subdomains/core/accounting/services/ledger-reconciliation.service.ts b/src/subdomains/core/accounting/services/ledger-reconciliation.service.ts index 5bcd015f5f..0af99790df 100644 --- a/src/subdomains/core/accounting/services/ledger-reconciliation.service.ts +++ b/src/subdomains/core/accounting/services/ledger-reconciliation.service.ts @@ -6,7 +6,7 @@ import { Asset } from 'src/shared/models/asset/asset.entity'; import { SettingService } from 'src/shared/models/setting/setting.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { LiquidityBalance } from 'src/subdomains/core/liquidity-management/entities/liquidity-balance.entity'; import { LiquidityManagementBalanceService } from 'src/subdomains/core/liquidity-management/services/liquidity-management-balance.service'; @@ -97,7 +97,7 @@ export class LedgerReconciliationService { private readonly refRewardService: RefRewardService, ) {} - @DfxCron(CronExpression.EVERY_DAY_AT_5AM, { process: Process.LEDGER_RECONCILIATION }) + @DfxCron(CronExpression.EVERY_DAY_AT_5AM, { scope: CronScope.Worker, process: Process.LEDGER_RECONCILIATION }) async run(): Promise { if (!(await this.jobService.isLedgerReady())) return; // cutover-gate (Blocker R1-6) diff --git a/src/subdomains/core/aml/services/sanction.service.ts b/src/subdomains/core/aml/services/sanction.service.ts index 2b1a5a08d0..835750654f 100644 --- a/src/subdomains/core/aml/services/sanction.service.ts +++ b/src/subdomains/core/aml/services/sanction.service.ts @@ -3,7 +3,7 @@ import { CronExpression } from '@nestjs/schedule'; import { Config, Environment } from 'src/config/config'; import { HttpService } from 'src/shared/services/http.service'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { Sanction } from '../entities/sanction.entity'; import { SanctionRepository } from '../repositories/sanction.repository'; @@ -40,7 +40,7 @@ export class SanctionService { ) {} // --- JOBS --- // - @DfxCron(CronExpression.EVERY_WEEKEND, { process: Process.SANCTION_SYNC }) + @DfxCron(CronExpression.EVERY_WEEKEND, { scope: CronScope.Worker, process: Process.SANCTION_SYNC }) async syncList() { const filePath = Config.environment === Environment.LOC ? this.fileName : `/home/${this.fileName}`; diff --git a/src/subdomains/core/buy-crypto/process/services/buy-crypto-job.service.ts b/src/subdomains/core/buy-crypto/process/services/buy-crypto-job.service.ts index d15e88ae8a..d11dd29009 100644 --- a/src/subdomains/core/buy-crypto/process/services/buy-crypto-job.service.ts +++ b/src/subdomains/core/buy-crypto/process/services/buy-crypto-job.service.ts @@ -1,7 +1,7 @@ import { Injectable } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; import { DisabledProcess, Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { BuyCryptoBatchService } from './buy-crypto-batch.service'; import { BuyCryptoDexService } from './buy-crypto-dex.service'; import { BuyCryptoNotificationService } from './buy-crypto-notification.service'; @@ -20,7 +20,7 @@ export class BuyCryptoJobService { private readonly buyCryptoPreparationService: BuyCryptoPreparationService, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.BUY_CRYPTO, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.BUY_CRYPTO, timeout: 7200 }) async process() { await this.buyCryptoRegistrationService.registerCryptoPayIn(); await this.buyCryptoRegistrationService.syncReturnTxId(); @@ -37,7 +37,11 @@ export class BuyCryptoJobService { await this.buyCryptoNotificationService.sendNotificationMails(); } - @DfxCron(CronExpression.EVERY_HOUR, { process: Process.BUY_CRYPTO_AGGREGATION, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_HOUR, { + scope: CronScope.Worker, + process: Process.BUY_CRYPTO_AGGREGATION, + timeout: 7200, + }) async checkAggregatingTransactions() { await this.buyCryptoPreparationService.checkAggregatingTransactions(); } diff --git a/src/subdomains/core/buy-crypto/routes/buy/buy.service.ts b/src/subdomains/core/buy-crypto/routes/buy/buy.service.ts index 3c05fd26db..26b467a4dc 100644 --- a/src/subdomains/core/buy-crypto/routes/buy/buy.service.ts +++ b/src/subdomains/core/buy-crypto/routes/buy/buy.service.ts @@ -15,7 +15,7 @@ import { AssetDtoMapper } from 'src/shared/models/asset/dto/asset-dto.mapper'; import { FiatDtoMapper } from 'src/shared/models/fiat/dto/fiat-dto.mapper'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { PaymentInfoService } from 'src/shared/services/payment-info.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { PdfUtil } from 'src/shared/utils/pdf.util'; import { Util } from 'src/shared/utils/util'; import { RouteService } from 'src/subdomains/core/route/route.service'; @@ -66,12 +66,12 @@ export class BuyService { ) {} // --- VOLUMES --- // - @DfxCron(CronExpression.EVERY_YEAR) + @DfxCron(CronExpression.EVERY_YEAR, { scope: CronScope.Worker }) async resetAnnualVolumes(): Promise { await this.buyRepo.update({ annualVolume: Not(0) }, { annualVolume: 0 }); } - @DfxCron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT) + @DfxCron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, { scope: CronScope.Worker }) async resetMonthlyVolumes(): Promise { await this.buyRepo.update({ monthlyVolume: Not(0) }, { monthlyVolume: 0 }); } diff --git a/src/subdomains/core/buy-crypto/routes/swap/swap.service.ts b/src/subdomains/core/buy-crypto/routes/swap/swap.service.ts index 64657b985e..9ff8f7b77f 100644 --- a/src/subdomains/core/buy-crypto/routes/swap/swap.service.ts +++ b/src/subdomains/core/buy-crypto/routes/swap/swap.service.ts @@ -17,7 +17,7 @@ import { Asset } from 'src/shared/models/asset/asset.entity'; import { AssetService } from 'src/shared/models/asset/asset.service'; import { AssetDtoMapper } from 'src/shared/models/asset/dto/asset-dto.mapper'; import { DfxLogger } from 'src/shared/services/dfx-logger'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { BuyCryptoExtended } from 'src/subdomains/core/history/mappers/transaction-dto.mapper'; import { RouteService } from 'src/subdomains/core/route/route.service'; @@ -85,12 +85,12 @@ export class SwapService { } // --- VOLUMES --- // - @DfxCron(CronExpression.EVERY_YEAR) + @DfxCron(CronExpression.EVERY_YEAR, { scope: CronScope.Worker }) async resetAnnualVolumes(): Promise { await this.swapRepo.update({ annualVolume: Not(0) }, { annualVolume: 0 }); } - @DfxCron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT) + @DfxCron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, { scope: CronScope.Worker }) async resetMonthlyVolumes(): Promise { await this.swapRepo.update({ monthlyVolume: Not(0) }, { monthlyVolume: 0 }); } diff --git a/src/subdomains/core/custody/services/custody-job.service.ts b/src/subdomains/core/custody/services/custody-job.service.ts index e1f3aa7d7f..b858a4ff04 100644 --- a/src/subdomains/core/custody/services/custody-job.service.ts +++ b/src/subdomains/core/custody/services/custody-job.service.ts @@ -8,7 +8,7 @@ import { OrderConfig } from '../config/order-config'; import { Config } from 'src/config/config'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { CustodyOrderStep } from '../entities/custody-order-step.entity'; import { @@ -38,14 +38,14 @@ export class CustodyJobService { private readonly custodyOrderService: CustodyOrderService, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.CUSTODY }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.CUSTODY }) async handleOrders() { await this.executeOrder(); await this.executeStep(); await this.checkStep(); } - @DfxCron(CronExpression.EVERY_DAY_AT_4AM, { process: Process.CUSTODY }) + @DfxCron(CronExpression.EVERY_DAY_AT_4AM, { scope: CronScope.Worker, process: Process.CUSTODY }) async resetExpiredConfirmedOrders() { const expiryDate = Util.daysBefore(Config.txRequestWaitingExpiryDays); diff --git a/src/subdomains/core/faucet-request/services/faucet-request.service.ts b/src/subdomains/core/faucet-request/services/faucet-request.service.ts index aecf71e7f7..b889faca64 100644 --- a/src/subdomains/core/faucet-request/services/faucet-request.service.ts +++ b/src/subdomains/core/faucet-request/services/faucet-request.service.ts @@ -13,7 +13,7 @@ import { AssetService } from 'src/shared/models/asset/asset.service'; import { AssetDtoMapper } from 'src/shared/models/asset/dto/asset-dto.mapper'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { KycLevel } from 'src/subdomains/generic/user/models/user-data/user-data.enum'; import { UserService } from 'src/subdomains/generic/user/models/user/user.service'; import { In, Not } from 'typeorm'; @@ -36,7 +36,7 @@ export class FaucetRequestService { return [Environment.DEV, Environment.LOC].includes(Config.environment) ? Blockchain.SEPOLIA : Blockchain.ETHEREUM; } - @DfxCron(CronExpression.EVERY_5_MINUTES, { process: Process.CRYPTO_PAYOUT }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.Worker, process: Process.CRYPTO_PAYOUT }) async checkFaucetRequests(): Promise { const pendingFaucets = await this.faucetRequestRepo.find({ where: { status: FaucetRequestStatus.IN_PROGRESS } }); for (const faucet of pendingFaucets) { diff --git a/src/subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.ts b/src/subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.ts index 8c74b457fe..9b77ca8cf1 100644 --- a/src/subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.ts +++ b/src/subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.ts @@ -2,7 +2,7 @@ import { BadRequestException, ConflictException, Injectable, NotFoundException } import { CronExpression } from '@nestjs/schedule'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { MailContext, MailType } from 'src/subdomains/supporting/notification/enums'; import { MailRequest } from 'src/subdomains/supporting/notification/interfaces'; @@ -77,7 +77,11 @@ export class LiquidityManagementPipelineService { //*** JOBS ***// - @DfxCron(CronExpression.EVERY_10_SECONDS, { process: Process.LIQUIDITY_MANAGEMENT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_10_SECONDS, { + scope: CronScope.Worker, + process: Process.LIQUIDITY_MANAGEMENT, + timeout: 1800, + }) async processPipelines(): Promise { let hasChanges = true; while (hasChanges) { diff --git a/src/subdomains/core/liquidity-management/services/liquidity-management-rule.service.ts b/src/subdomains/core/liquidity-management/services/liquidity-management-rule.service.ts index 0a6c68d992..93e1af5913 100644 --- a/src/subdomains/core/liquidity-management/services/liquidity-management-rule.service.ts +++ b/src/subdomains/core/liquidity-management/services/liquidity-management-rule.service.ts @@ -6,7 +6,7 @@ import { Fiat } from 'src/shared/models/fiat/fiat.entity'; import { FiatService } from 'src/shared/models/fiat/fiat.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { MailContext, MailType } from 'src/subdomains/supporting/notification/enums'; import { MailRequest } from 'src/subdomains/supporting/notification/interfaces'; import { NotificationService } from 'src/subdomains/supporting/notification/services/notification.service'; @@ -105,7 +105,11 @@ export class LiquidityManagementRuleService { //*** JOBS ***// - @DfxCron(CronExpression.EVERY_5_MINUTES, { process: Process.LIQUIDITY_MANAGEMENT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { + scope: CronScope.Worker, + process: Process.LIQUIDITY_MANAGEMENT, + timeout: 1800, + }) async reactivateRules(): Promise { const rules = await this.ruleRepo.findBy({ status: LiquidityManagementRuleStatus.PAUSED, diff --git a/src/subdomains/core/liquidity-management/services/liquidity-management.service.ts b/src/subdomains/core/liquidity-management/services/liquidity-management.service.ts index 3559b5631f..32afddae8e 100644 --- a/src/subdomains/core/liquidity-management/services/liquidity-management.service.ts +++ b/src/subdomains/core/liquidity-management/services/liquidity-management.service.ts @@ -4,7 +4,7 @@ import { Config } from 'src/config/config'; import { SettingService } from 'src/shared/models/setting/setting.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { DisabledProcess, Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { PriceCurrency, @@ -37,7 +37,11 @@ export class LiquidityManagementService { //*** JOBS ***// - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.LIQUIDITY_MANAGEMENT_CHECK_BALANCES, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { + scope: CronScope.Worker, + process: Process.LIQUIDITY_MANAGEMENT_CHECK_BALANCES, + timeout: 1800, + }) async checkLiquidityBalances() { const rules = await this.ruleRepo.findBy({ status: Not(LiquidityManagementRuleStatus.DISABLED) }); const balances = await this.balanceService.refreshBalances(rules); diff --git a/src/subdomains/core/monitoring/observers/aml.observer.ts b/src/subdomains/core/monitoring/observers/aml.observer.ts index b23e43ea9e..050c6f80e5 100644 --- a/src/subdomains/core/monitoring/observers/aml.observer.ts +++ b/src/subdomains/core/monitoring/observers/aml.observer.ts @@ -3,7 +3,7 @@ import { CronExpression } from '@nestjs/schedule'; import { RepositoryFactory } from 'src/shared/repositories/repository.factory'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { MetricObserver } from 'src/subdomains/core/monitoring/metric.observer'; import { MonitoringService } from 'src/subdomains/core/monitoring/monitoring.service'; import { IsNull } from 'typeorm'; @@ -31,7 +31,7 @@ export class AmlObserver extends MetricObserver { super(monitoringService, 'payment', 'aml'); } - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.MONITORING, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.MONITORING, timeout: 1800 }) async fetch() { const data = await this.getAmlData(); diff --git a/src/subdomains/core/monitoring/observers/bank.observer.ts b/src/subdomains/core/monitoring/observers/bank.observer.ts index 622854b9ce..342338fc84 100644 --- a/src/subdomains/core/monitoring/observers/bank.observer.ts +++ b/src/subdomains/core/monitoring/observers/bank.observer.ts @@ -7,7 +7,7 @@ import { YapealService } from 'src/integration/bank/services/yapeal.service'; import { RepositoryFactory } from 'src/shared/repositories/repository.factory'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { MetricObserver } from 'src/subdomains/core/monitoring/metric.observer'; import { MonitoringService } from 'src/subdomains/core/monitoring/monitoring.service'; @@ -38,7 +38,7 @@ export class BankObserver extends MetricObserver { super(monitoringService, 'bank', 'balance'); } - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.MONITORING, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.MONITORING, timeout: 1800 }) async fetch() { let data = []; diff --git a/src/subdomains/core/monitoring/observers/checkout.observer.ts b/src/subdomains/core/monitoring/observers/checkout.observer.ts index cf1f5e0f15..3bba38385e 100644 --- a/src/subdomains/core/monitoring/observers/checkout.observer.ts +++ b/src/subdomains/core/monitoring/observers/checkout.observer.ts @@ -3,7 +3,7 @@ import { CronExpression } from '@nestjs/schedule'; import { CheckoutBalances, CheckoutService } from 'src/integration/checkout/services/checkout.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { MetricObserver } from 'src/subdomains/core/monitoring/metric.observer'; import { MonitoringService } from 'src/subdomains/core/monitoring/monitoring.service'; import { CheckoutTxService } from 'src/subdomains/supporting/fiat-payin/services/checkout-tx.service'; @@ -30,7 +30,7 @@ export class CheckoutObserver extends MetricObserver { super(monitoringService, 'checkout', 'balance'); } - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.MONITORING, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.MONITORING, timeout: 1800 }) async fetch() { if (!this.checkoutService.isAvailable()) { if (!this.unavailableWarningLogged) { diff --git a/src/subdomains/core/monitoring/observers/exchange.observer.ts b/src/subdomains/core/monitoring/observers/exchange.observer.ts index 66ae0e7ff3..8d473fa1c7 100644 --- a/src/subdomains/core/monitoring/observers/exchange.observer.ts +++ b/src/subdomains/core/monitoring/observers/exchange.observer.ts @@ -5,7 +5,7 @@ import { ExchangeName } from 'src/integration/exchange/enums/exchange.enum'; import { RepositoryFactory } from 'src/shared/repositories/repository.factory'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { DisabledProcess, Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { MetricObserver } from 'src/subdomains/core/monitoring/metric.observer'; import { MonitoringService } from 'src/subdomains/core/monitoring/monitoring.service'; @@ -27,7 +27,7 @@ export class ExchangeObserver extends MetricObserver { super(monitoringService, 'exchange', 'volume'); } - @DfxCron(CronExpression.EVERY_10_MINUTES, { process: Process.MONITORING, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.MONITORING, timeout: 1800 }) async fetch() { if (DisabledProcess(Process.MONITORING)) return; diff --git a/src/subdomains/core/monitoring/observers/external-services.observer.ts b/src/subdomains/core/monitoring/observers/external-services.observer.ts index 9f36d27b65..815b6440c3 100644 --- a/src/subdomains/core/monitoring/observers/external-services.observer.ts +++ b/src/subdomains/core/monitoring/observers/external-services.observer.ts @@ -4,7 +4,7 @@ import { IbanService } from 'src/integration/bank/services/iban.service'; import { LetterService } from 'src/integration/letter/letter.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { MetricObserver } from 'src/subdomains/core/monitoring/metric.observer'; import { MonitoringService } from 'src/subdomains/core/monitoring/monitoring.service'; @@ -31,7 +31,7 @@ export class ExternalServicesObserver extends MetricObserver { super(monitoringService, 'liquidity', 'trading'); } - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.MONITORING, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.MONITORING, timeout: 1800 }) async fetch() { const data = await this.getLiquidityData(); diff --git a/src/subdomains/core/monitoring/observers/node-balance.observer.ts b/src/subdomains/core/monitoring/observers/node-balance.observer.ts index 906f98fc67..b66f03c017 100644 --- a/src/subdomains/core/monitoring/observers/node-balance.observer.ts +++ b/src/subdomains/core/monitoring/observers/node-balance.observer.ts @@ -4,7 +4,7 @@ import { BitcoinClient } from 'src/integration/blockchain/bitcoin/node/bitcoin-c import { BitcoinNodeType, BitcoinService } from 'src/integration/blockchain/bitcoin/services/bitcoin.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { MetricObserver } from 'src/subdomains/core/monitoring/metric.observer'; import { MonitoringService } from 'src/subdomains/core/monitoring/monitoring.service'; @@ -31,7 +31,7 @@ export class NodeBalanceObserver extends MetricObserver { this.bitcoinClient = bitcoinService.getDefaultClient(BitcoinNodeType.BTC_INPUT); } - @DfxCron(CronExpression.EVERY_10_MINUTES, { process: Process.MONITORING, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.MONITORING, timeout: 1800 }) async fetch(): Promise { const data = await this.getNode(); diff --git a/src/subdomains/core/monitoring/observers/node-health.observer.ts b/src/subdomains/core/monitoring/observers/node-health.observer.ts index 78b5f37adf..f6462ea8c8 100644 --- a/src/subdomains/core/monitoring/observers/node-health.observer.ts +++ b/src/subdomains/core/monitoring/observers/node-health.observer.ts @@ -3,7 +3,7 @@ import { CronExpression } from '@nestjs/schedule'; import { BitcoinNodeType, BitcoinService } from 'src/integration/blockchain/bitcoin/services/bitcoin.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { MetricObserver } from 'src/subdomains/core/monitoring/metric.observer'; import { MonitoringService } from 'src/subdomains/core/monitoring/monitoring.service'; @@ -45,7 +45,7 @@ export class NodeHealthObserver extends MetricObserver { this.emit(data); } - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.MONITORING, timeout: 360 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.MONITORING, timeout: 360 }) async fetch(): Promise { const previousState = this.data; diff --git a/src/subdomains/core/monitoring/observers/payment.observer.ts b/src/subdomains/core/monitoring/observers/payment.observer.ts index bbcbf820d7..ecbefe690b 100644 --- a/src/subdomains/core/monitoring/observers/payment.observer.ts +++ b/src/subdomains/core/monitoring/observers/payment.observer.ts @@ -4,7 +4,7 @@ import { FRICK_TERMINAL_STATES } from 'src/integration/bank/dto/frick.dto'; import { RepositoryFactory } from 'src/shared/repositories/repository.factory'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { MetricObserver } from 'src/subdomains/core/monitoring/metric.observer'; import { MonitoringService } from 'src/subdomains/core/monitoring/monitoring.service'; @@ -51,7 +51,7 @@ export class PaymentObserver extends MetricObserver { super(monitoringService, 'payment', 'combined'); } - @DfxCron(CronExpression.EVERY_10_MINUTES, { process: Process.MONITORING, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.MONITORING, timeout: 1800 }) async fetch() { const data = await this.getPayment(); diff --git a/src/subdomains/core/monitoring/observers/realunit-w2w-gas.observer.ts b/src/subdomains/core/monitoring/observers/realunit-w2w-gas.observer.ts index 8f38a447ce..b14f1167d0 100644 --- a/src/subdomains/core/monitoring/observers/realunit-w2w-gas.observer.ts +++ b/src/subdomains/core/monitoring/observers/realunit-w2w-gas.observer.ts @@ -6,7 +6,7 @@ import { SepoliaService } from 'src/integration/blockchain/sepolia/sepolia.servi import { EvmClient } from 'src/integration/blockchain/shared/evm/evm-client'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { MetricObserver } from 'src/subdomains/core/monitoring/metric.observer'; import { MonitoringService } from 'src/subdomains/core/monitoring/monitoring.service'; import { MailContext, MailType } from 'src/subdomains/supporting/notification/enums'; @@ -40,7 +40,7 @@ export class RealUnitW2wGasObserver extends MetricObserver { super(monitoringService, 'realUnit', 'w2wGasBalance'); } - @DfxCron(CronExpression.EVERY_10_MINUTES, { process: Process.MONITORING, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.MONITORING, timeout: 1800 }) async fetch(): Promise { const data = await this.getData(); diff --git a/src/subdomains/core/monitoring/observers/user.observer.ts b/src/subdomains/core/monitoring/observers/user.observer.ts index 815d84d5f7..1a98cd006d 100644 --- a/src/subdomains/core/monitoring/observers/user.observer.ts +++ b/src/subdomains/core/monitoring/observers/user.observer.ts @@ -3,7 +3,7 @@ import { CronExpression } from '@nestjs/schedule'; import { RepositoryFactory } from 'src/shared/repositories/repository.factory'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { MetricObserver } from 'src/subdomains/core/monitoring/metric.observer'; import { MonitoringService } from 'src/subdomains/core/monitoring/monitoring.service'; import { IsNull } from 'typeorm'; @@ -27,7 +27,7 @@ export class UserObserver extends MetricObserver { super(monitoringService, 'user', 'kyc'); } - @DfxCron(CronExpression.EVERY_10_MINUTES, { process: Process.MONITORING, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.MONITORING, timeout: 1800 }) async fetch(): Promise { const data = await this.getUser(); diff --git a/src/subdomains/core/payment-link/services/payment-cron.service.ts b/src/subdomains/core/payment-link/services/payment-cron.service.ts index 08d9fdb93c..6a2f20e8a3 100644 --- a/src/subdomains/core/payment-link/services/payment-cron.service.ts +++ b/src/subdomains/core/payment-link/services/payment-cron.service.ts @@ -1,7 +1,7 @@ import { Injectable } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { PaymentActivationService } from './payment-activation.service'; import { PaymentBalanceService } from './payment-balance.service'; import { PaymentLinkPaymentService } from './payment-link-payment.service'; @@ -16,19 +16,19 @@ export class PaymentCronService { private readonly paymentBalanceService: PaymentBalanceService, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.PAYMENT_EXPIRATION }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.PAYMENT_EXPIRATION }) async processExpiredPayments(): Promise { await this.paymentLinkPaymentService.processExpiredPayments(); await this.paymentActivationService.processExpiredActivations(); await this.paymentQuoteService.processExpiredQuotes(); } - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.PAYMENT_CONFIRMATIONS }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.PAYMENT_CONFIRMATIONS }) async checkTxConfirmations(): Promise { await this.paymentLinkPaymentService.checkTxConfirmations(); } - @DfxCron(CronExpression.EVERY_HOUR, { process: Process.PAYMENT_FORWARDING }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, process: Process.PAYMENT_FORWARDING }) async forwardDeposits(): Promise { await this.paymentBalanceService.forwardDeposits(); } diff --git a/src/subdomains/core/referral/process/ref.service.ts b/src/subdomains/core/referral/process/ref.service.ts index c952dbe942..87fc7edc61 100644 --- a/src/subdomains/core/referral/process/ref.service.ts +++ b/src/subdomains/core/referral/process/ref.service.ts @@ -1,7 +1,7 @@ import { Injectable } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; import { DfxLogger } from 'src/shared/services/dfx-logger'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { IsNull, LessThan } from 'typeorm'; import { Ref } from './ref.entity'; @@ -15,7 +15,7 @@ export class RefService { constructor(private readonly repo: RefRepository) {} - @DfxCron(CronExpression.EVERY_HOUR, { timeout: 7200 }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, timeout: 7200 }) async checkRefs(): Promise { const expirationDate = Util.daysBefore(this.refExpirationDays); diff --git a/src/subdomains/core/referral/reward/services/ref-reward-job.service.ts b/src/subdomains/core/referral/reward/services/ref-reward-job.service.ts index 573a547e8f..6863b0972f 100644 --- a/src/subdomains/core/referral/reward/services/ref-reward-job.service.ts +++ b/src/subdomains/core/referral/reward/services/ref-reward-job.service.ts @@ -1,7 +1,7 @@ import { Injectable } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { RefRewardDexService } from './ref-reward-dex.service'; import { RefRewardNotificationService } from './ref-reward-notification.service'; import { RefRewardOutService } from './ref-reward-out.service'; @@ -16,12 +16,12 @@ export class RefRewardJobService { private readonly refRewardService: RefRewardService, ) {} - @DfxCron(CronExpression.EVERY_DAY_AT_6AM, { process: Process.REF_PAYOUT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_DAY_AT_6AM, { scope: CronScope.Worker, process: Process.REF_PAYOUT, timeout: 1800 }) async createPendingRefRewards() { await this.refRewardService.createPendingRefRewards(); } - @DfxCron(CronExpression.EVERY_10_MINUTES, { process: Process.REF_PAYOUT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.REF_PAYOUT, timeout: 1800 }) async processPendingRefRewards() { await this.refRewardDexService.secureLiquidity(); await this.refRewardOutService.checkPaidTransaction(); diff --git a/src/subdomains/core/sell-crypto/process/services/buy-fiat-job.service.ts b/src/subdomains/core/sell-crypto/process/services/buy-fiat-job.service.ts index f788e47664..55ef430b2c 100644 --- a/src/subdomains/core/sell-crypto/process/services/buy-fiat-job.service.ts +++ b/src/subdomains/core/sell-crypto/process/services/buy-fiat-job.service.ts @@ -1,7 +1,7 @@ import { Injectable } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; import { DisabledProcess, Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { BuyFiatPreparationService } from './buy-fiat-preparation.service'; import { BuyFiatRegistrationService } from './buy-fiat-registration.service'; @@ -12,7 +12,7 @@ export class BuyFiatJobService { private readonly buyFiatPreparationService: BuyFiatPreparationService, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.BUY_FIAT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.BUY_FIAT, timeout: 1800 }) async checkCryptoPayIn() { await this.buyFiatRegistrationService.registerSellPayIn(); await this.buyFiatRegistrationService.syncReturnTxId(); @@ -26,7 +26,7 @@ export class BuyFiatJobService { await this.buyFiatPreparationService.chargebackTx(); } - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.BUY_FIAT, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.BUY_FIAT, timeout: 7200 }) async addFiatOutputs(): Promise { await this.buyFiatPreparationService.addFiatOutputs(); } diff --git a/src/subdomains/core/sell-crypto/process/services/buy-fiat-notification.service.ts b/src/subdomains/core/sell-crypto/process/services/buy-fiat-notification.service.ts index 8f110b2f79..7835b4ecfa 100644 --- a/src/subdomains/core/sell-crypto/process/services/buy-fiat-notification.service.ts +++ b/src/subdomains/core/sell-crypto/process/services/buy-fiat-notification.service.ts @@ -2,7 +2,7 @@ import { Injectable } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { AmlReason, AmlReasonWithoutReason, KycAmlReasons } from 'src/subdomains/core/aml/enums/aml-reason.enum'; import { MailContext, MailType } from 'src/subdomains/supporting/notification/enums'; @@ -30,7 +30,7 @@ export class BuyFiatNotificationService { private readonly notificationService: NotificationService, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.BUY_FIAT_MAIL, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.BUY_FIAT_MAIL, timeout: 1800 }) async sendNotificationMails(): Promise { await this.paymentCompleted(); await this.chargebackInitiated(); diff --git a/src/subdomains/core/sell-crypto/route/sell.service.ts b/src/subdomains/core/sell-crypto/route/sell.service.ts index 81117b5174..3b52153664 100644 --- a/src/subdomains/core/sell-crypto/route/sell.service.ts +++ b/src/subdomains/core/sell-crypto/route/sell.service.ts @@ -17,7 +17,7 @@ import { AssetService } from 'src/shared/models/asset/asset.service'; import { AssetDtoMapper } from 'src/shared/models/asset/dto/asset-dto.mapper'; import { FiatDtoMapper } from 'src/shared/models/fiat/dto/fiat-dto.mapper'; import { DfxLogger } from 'src/shared/services/dfx-logger'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { CreateSellDto } from 'src/subdomains/core/sell-crypto/route/dto/create-sell.dto'; import { UpdateSellDto } from 'src/subdomains/core/sell-crypto/route/dto/update-sell.dto'; @@ -223,12 +223,12 @@ export class SellService { } // --- VOLUMES --- // - @DfxCron(CronExpression.EVERY_YEAR) + @DfxCron(CronExpression.EVERY_YEAR, { scope: CronScope.Worker }) async resetAnnualVolumes(): Promise { await this.sellRepo.update({ annualVolume: Not(0) }, { annualVolume: 0 }); } - @DfxCron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT) + @DfxCron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, { scope: CronScope.Worker }) async resetMonthlyVolumes(): Promise { await this.sellRepo.update({ monthlyVolume: Not(0) }, { monthlyVolume: 0 }); } diff --git a/src/subdomains/core/trading/services/trading-job.service.ts b/src/subdomains/core/trading/services/trading-job.service.ts index af6cb02102..200e706706 100644 --- a/src/subdomains/core/trading/services/trading-job.service.ts +++ b/src/subdomains/core/trading/services/trading-job.service.ts @@ -1,7 +1,7 @@ import { Injectable } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { TradingOrderService } from './trading-order.service'; import { TradingRuleService } from './trading-rule.service'; @@ -14,19 +14,19 @@ export class TradingJobService { // --- RULES --- // - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.TRADING, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.TRADING, timeout: 1800 }) async processRules() { await this.ruleService.processRules(); } - @DfxCron(CronExpression.EVERY_5_MINUTES, { process: Process.TRADING, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.Worker, process: Process.TRADING, timeout: 1800 }) async reactivateRules(): Promise { await this.ruleService.reactivateRules(); } // --- ORDERS --- // - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.TRADING, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.TRADING, timeout: 1800 }) async processOrders() { await this.orderService.processOrders(); } diff --git a/src/subdomains/generic/admin/admin.service.ts b/src/subdomains/generic/admin/admin.service.ts index 34db9b377f..2182b1b516 100644 --- a/src/subdomains/generic/admin/admin.service.ts +++ b/src/subdomains/generic/admin/admin.service.ts @@ -10,7 +10,7 @@ import { EvmBlockchains } from 'src/integration/blockchain/shared/util/blockchai import { AssetService } from 'src/shared/models/asset/asset.service'; import { SettingService } from 'src/shared/models/setting/setting.service'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { LiquidityOrderContext } from 'src/subdomains/supporting/dex/entities/liquidity-order.entity'; import { ReserveLiquidityRequest } from 'src/subdomains/supporting/dex/interfaces'; import { DexService } from 'src/subdomains/supporting/dex/services/dex.service'; @@ -79,7 +79,7 @@ export class AdminService { } } - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.PAY_OUT, timeout: 3600 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.PAY_OUT, timeout: 3600 }) async completeLiquidityOrders() { for (const context of Object.values(PayoutRequestContext)) { const lContext = context as unknown as LiquidityOrderContext; diff --git a/src/subdomains/generic/kyc/services/kyc-notification.service.ts b/src/subdomains/generic/kyc/services/kyc-notification.service.ts index a9803e30a0..9f14ea617c 100644 --- a/src/subdomains/generic/kyc/services/kyc-notification.service.ts +++ b/src/subdomains/generic/kyc/services/kyc-notification.service.ts @@ -3,7 +3,7 @@ import { CronExpression } from '@nestjs/schedule'; import { Config } from 'src/config/config'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { DisabledProcess, Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { MailContext, MailType } from 'src/subdomains/supporting/notification/enums'; import { MailKey, MailTranslationKey } from 'src/subdomains/supporting/notification/factories/mail.factory'; @@ -26,7 +26,7 @@ export class KycNotificationService { private readonly webhookService: WebhookService, ) {} - @DfxCron(CronExpression.EVERY_HOUR, { process: Process.KYC_MAIL, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, process: Process.KYC_MAIL, timeout: 1800 }) async sendNotificationMails(): Promise { await this.autoKycStepReminder(); } diff --git a/src/subdomains/generic/kyc/services/kyc.service.ts b/src/subdomains/generic/kyc/services/kyc.service.ts index 92de4fa10f..8e742fc32b 100644 --- a/src/subdomains/generic/kyc/services/kyc.service.ts +++ b/src/subdomains/generic/kyc/services/kyc.service.ts @@ -20,7 +20,7 @@ import { IEntity, UpdateResult } from 'src/shared/models/entity'; import { LanguageService } from 'src/shared/models/language/language.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { DisabledProcess, Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { QueueHandler } from 'src/shared/utils/queue-handler'; import { Util } from 'src/shared/utils/util'; import { CheckStatus } from 'src/subdomains/core/aml/enums/check-status.enum'; @@ -134,7 +134,7 @@ export class KycService { this.webhookQueue = new QueueHandler(); } - @DfxCron(CronExpression.EVERY_DAY_AT_4AM, { process: Process.KYC }) + @DfxCron(CronExpression.EVERY_DAY_AT_4AM, { scope: CronScope.Worker, process: Process.KYC }) async checkIdentSteps(): Promise { const expiredIdentSteps = await this.kycStepRepo.find({ where: { @@ -162,7 +162,7 @@ export class KycService { } } - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.KYC }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.KYC }) async reviewKycSteps(): Promise { await this.reviewNationalityStep(); await this.reviewIdentSteps(); diff --git a/src/subdomains/generic/user/models/bank-data/bank-data.service.ts b/src/subdomains/generic/user/models/bank-data/bank-data.service.ts index 538362cc63..cb5f273823 100644 --- a/src/subdomains/generic/user/models/bank-data/bank-data.service.ts +++ b/src/subdomains/generic/user/models/bank-data/bank-data.service.ts @@ -6,7 +6,7 @@ import { CountryService } from 'src/shared/models/country/country.service'; import { FiatService } from 'src/shared/models/fiat/fiat.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { DisabledProcess, Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { KycStepName } from 'src/subdomains/generic/kyc/enums/kyc-step-name.enum'; import { ReviewStatus } from 'src/subdomains/generic/kyc/enums/review-status.enum'; @@ -46,7 +46,11 @@ export class BankDataService { private readonly kycAdminService: KycAdminService, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.BANK_DATA_VERIFICATION, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { + scope: CronScope.Worker, + process: Process.BANK_DATA_VERIFICATION, + timeout: 1800, + }) async checkAndSetActive() { await this.checkUnverifiedBankDatas(); } diff --git a/src/subdomains/generic/user/models/organization/organization.service.ts b/src/subdomains/generic/user/models/organization/organization.service.ts index 97d4fa2d9c..e0196eff70 100644 --- a/src/subdomains/generic/user/models/organization/organization.service.ts +++ b/src/subdomains/generic/user/models/organization/organization.service.ts @@ -3,7 +3,7 @@ import { CronExpression } from '@nestjs/schedule'; import { CountryService } from 'src/shared/models/country/country.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { In, IsNull } from 'typeorm'; import { AccountType } from '../user-data/account-type.enum'; import { UserDataRepository } from '../user-data/user-data.repository'; @@ -21,7 +21,7 @@ export class OrganizationService { private readonly userDataRepo: UserDataRepository, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.ORGANIZATION_SYNC, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.ORGANIZATION_SYNC, timeout: 1800 }) async syncOrganization() { const entities = await this.userDataRepo.findBy({ organization: { id: IsNull() }, diff --git a/src/subdomains/generic/user/models/user-data/jwt-revocation-sync.service.ts b/src/subdomains/generic/user/models/user-data/jwt-revocation-sync.service.ts index 7a50409395..9cc38c1c89 100644 --- a/src/subdomains/generic/user/models/user-data/jwt-revocation-sync.service.ts +++ b/src/subdomains/generic/user/models/user-data/jwt-revocation-sync.service.ts @@ -1,7 +1,7 @@ import { Injectable } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; import { SettingService } from 'src/shared/models/setting/setting.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { In } from 'typeorm'; import { RiskStatus, UserDataStatus } from './user-data.enum'; import { UserDataRepository } from './user-data.repository'; @@ -25,7 +25,7 @@ export class JwtRevocationSyncService { // Runs every minute: fast revocation of a blocked or compromised account is a security requirement that // warrants the security-revocation exception to the "prefer 15min" cron guideline. - @DfxCron(CronExpression.EVERY_MINUTE, { timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, timeout: 1800 }) async syncDeniedJwtAccounts(): Promise { const blockedAccounts = await this.userDataRepo.find({ select: { id: true }, diff --git a/src/subdomains/generic/user/models/user-data/user-data-job.service.ts b/src/subdomains/generic/user/models/user-data/user-data-job.service.ts index 4cfa08916a..debd2014a9 100644 --- a/src/subdomains/generic/user/models/user-data/user-data-job.service.ts +++ b/src/subdomains/generic/user/models/user-data/user-data-job.service.ts @@ -1,7 +1,7 @@ import { Injectable } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { CheckStatus } from 'src/subdomains/core/aml/enums/check-status.enum'; import { FileType } from 'src/subdomains/generic/kyc/dto/kyc-file.dto'; import { KycStepName } from 'src/subdomains/generic/kyc/enums/kyc-step-name.enum'; @@ -15,7 +15,7 @@ import { UserDataRepository } from './user-data.repository'; export class UserDataJobService { constructor(private readonly userDataRepo: UserDataRepository) {} - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.USER_DATA, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.USER_DATA, timeout: 1800 }) async fillUserData() { await this.bankTxVerification(); await this.setAccountOpener(); diff --git a/src/subdomains/generic/user/models/user-data/user-data-notification.service.ts b/src/subdomains/generic/user/models/user-data/user-data-notification.service.ts index d02f72499c..e403f6063c 100644 --- a/src/subdomains/generic/user/models/user-data/user-data-notification.service.ts +++ b/src/subdomains/generic/user/models/user-data/user-data-notification.service.ts @@ -3,7 +3,7 @@ import { CronExpression } from '@nestjs/schedule'; import { Config } from 'src/config/config'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { MailContext, MailType } from 'src/subdomains/supporting/notification/enums'; import { MailKey, MailTranslationKey } from 'src/subdomains/supporting/notification/factories/mail.factory'; @@ -20,7 +20,7 @@ export class UserDataNotificationService { private readonly notificationService: NotificationService, ) {} - @DfxCron(CronExpression.EVERY_HOUR, { process: Process.BLACK_SQUAD_MAIL, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, process: Process.BLACK_SQUAD_MAIL, timeout: 1800 }) async sendNotificationMails(): Promise { await this.blackSquadInvitation(); } diff --git a/src/subdomains/generic/user/models/user-data/user-data.service.ts b/src/subdomains/generic/user/models/user-data/user-data.service.ts index 6cc4a9e281..20f459a7fe 100644 --- a/src/subdomains/generic/user/models/user-data/user-data.service.ts +++ b/src/subdomains/generic/user/models/user-data/user-data.service.ts @@ -1191,7 +1191,7 @@ export class UserDataService { } // --- VOLUMES --- // - @DfxCron(CronExpression.EVERY_YEAR) + @DfxCron(CronExpression.EVERY_YEAR, { scope: CronScope.Worker }) async resetAnnualVolumes(): Promise { await this.userDataRepo.update( [{ annualBuyVolume: Not(0) }, { annualSellVolume: Not(0) }, { annualCryptoVolume: Not(0) }], @@ -1199,7 +1199,7 @@ export class UserDataService { ); } - @DfxCron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT) + @DfxCron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, { scope: CronScope.Worker }) async resetMonthlyVolumes(): Promise { await this.userDataRepo.update( [{ monthlyBuyVolume: Not(0) }, { monthlySellVolume: Not(0) }, { monthlyCryptoVolume: Not(0) }], diff --git a/src/subdomains/generic/user/models/user/user-job.service.ts b/src/subdomains/generic/user/models/user/user-job.service.ts index a0b91e43ea..abd2112b9b 100644 --- a/src/subdomains/generic/user/models/user/user-job.service.ts +++ b/src/subdomains/generic/user/models/user/user-job.service.ts @@ -1,7 +1,7 @@ import { Injectable } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { FileType } from 'src/subdomains/generic/kyc/dto/kyc-file.dto'; import { IsNull, Like, MoreThan } from 'typeorm'; import { UserRepository } from './user.repository'; @@ -10,7 +10,7 @@ import { UserRepository } from './user.repository'; export class UserJobService { constructor(private readonly userRepo: UserRepository) {} - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.USER, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.USER, timeout: 1800 }) async fillUser() { await this.approveUser(); } diff --git a/src/subdomains/generic/user/models/user/user.service.ts b/src/subdomains/generic/user/models/user/user.service.ts index b5d4d3d72e..d4a5ddd17d 100644 --- a/src/subdomains/generic/user/models/user/user.service.ts +++ b/src/subdomains/generic/user/models/user/user.service.ts @@ -20,7 +20,7 @@ import { LanguageDtoMapper } from 'src/shared/models/language/dto/language-dto.m import { LanguageService } from 'src/shared/models/language/language.service'; import { ApiKeyService } from 'src/shared/services/api-key.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { CheckStatus } from 'src/subdomains/core/aml/enums/check-status.enum'; import { HistoryFilter, HistoryFilterKey } from 'src/subdomains/core/history/dto/history-filter.dto'; @@ -541,7 +541,7 @@ export class UserService { } // --- VOLUMES --- // - @DfxCron(CronExpression.EVERY_YEAR) + @DfxCron(CronExpression.EVERY_YEAR, { scope: CronScope.Worker }) async resetAnnualVolumes(): Promise { await this.userRepo.update( [{ annualBuyVolume: Not(0) }, { annualSellVolume: Not(0) }, { annualCryptoVolume: Not(0) }], @@ -549,7 +549,7 @@ export class UserService { ); } - @DfxCron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT) + @DfxCron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, { scope: CronScope.Worker }) async resetMonthlyVolumes(): Promise { await this.userRepo.update( [{ monthlyBuyVolume: Not(0) }, { monthlySellVolume: Not(0) }, { monthlyCryptoVolume: Not(0) }], diff --git a/src/subdomains/generic/user/services/webhook/webhook-notification.service.ts b/src/subdomains/generic/user/services/webhook/webhook-notification.service.ts index 2c0570d69a..7eb14d7b9f 100644 --- a/src/subdomains/generic/user/services/webhook/webhook-notification.service.ts +++ b/src/subdomains/generic/user/services/webhook/webhook-notification.service.ts @@ -3,7 +3,7 @@ import { CronExpression } from '@nestjs/schedule'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { HttpService } from 'src/shared/services/http.service'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { MailContext, MailType } from 'src/subdomains/supporting/notification/enums'; import { NotificationService } from 'src/subdomains/supporting/notification/services/notification.service'; import { IsNull } from 'typeorm'; @@ -23,7 +23,7 @@ export class WebhookNotificationService { private readonly notificationService: NotificationService, ) {} - @DfxCron(CronExpression.EVERY_5_MINUTES, { process: Process.WEBHOOK, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.Worker, process: Process.WEBHOOK, timeout: 1800 }) async sendWebhooks() { await this.sendOpenWebhooks(); } diff --git a/src/subdomains/supporting/bank-tx/bank-tx-return/bank-tx-return-notification.service.ts b/src/subdomains/supporting/bank-tx/bank-tx-return/bank-tx-return-notification.service.ts index 253a7e4eaf..a27f851d94 100644 --- a/src/subdomains/supporting/bank-tx/bank-tx-return/bank-tx-return-notification.service.ts +++ b/src/subdomains/supporting/bank-tx/bank-tx-return/bank-tx-return-notification.service.ts @@ -2,7 +2,7 @@ import { Injectable } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { IsNull, Not } from 'typeorm'; import { MailContext, MailType } from '../../notification/enums'; import { MailKey, MailTranslationKey } from '../../notification/factories/mail.factory'; @@ -18,7 +18,11 @@ export class BankTxReturnNotificationService { private readonly notificationService: NotificationService, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.BANK_TX_RETURN_MAIL, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { + scope: CronScope.Worker, + process: Process.BANK_TX_RETURN_MAIL, + timeout: 1800, + }) async sendBankTxReturnMail() { await this.chargebackInitiated(); } diff --git a/src/subdomains/supporting/bank-tx/bank-tx-return/bank-tx-return.service.ts b/src/subdomains/supporting/bank-tx/bank-tx-return/bank-tx-return.service.ts index 9a98488e3c..f14ef8b59d 100644 --- a/src/subdomains/supporting/bank-tx/bank-tx-return/bank-tx-return.service.ts +++ b/src/subdomains/supporting/bank-tx/bank-tx-return/bank-tx-return.service.ts @@ -3,7 +3,7 @@ import { CronExpression } from '@nestjs/schedule'; import { FiatService } from 'src/shared/models/fiat/fiat.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { BankTxRefund, RefundInternalDto } from 'src/subdomains/core/history/dto/refund-internal.dto'; import { TransactionUtilService } from 'src/subdomains/core/transaction/transaction-util.service'; @@ -35,7 +35,7 @@ export class BankTxReturnService { private readonly fiatService: FiatService, ) {} - @DfxCron(CronExpression.EVERY_5_MINUTES, { process: Process.BANK_TX_RETURN, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.Worker, process: Process.BANK_TX_RETURN, timeout: 1800 }) async fillBankTxReturn() { await this.chargebackTx(); await this.setFiatAmounts(); diff --git a/src/subdomains/supporting/bank-tx/bank-tx/services/bank-tx.service.ts b/src/subdomains/supporting/bank-tx/bank-tx/services/bank-tx.service.ts index 2853cdb53f..6f67bb8871 100644 --- a/src/subdomains/supporting/bank-tx/bank-tx/services/bank-tx.service.ts +++ b/src/subdomains/supporting/bank-tx/bank-tx/services/bank-tx.service.ts @@ -15,7 +15,7 @@ import { FiatService } from 'src/shared/models/fiat/fiat.service'; import { SettingService } from 'src/shared/models/setting/setting.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { AmountType, Util } from 'src/shared/utils/util'; import { BuyCryptoService } from 'src/subdomains/core/buy-crypto/process/services/buy-crypto.service'; import { BuyService } from 'src/subdomains/core/buy-crypto/routes/buy/buy.service'; @@ -132,7 +132,7 @@ export class BankTxService implements OnModuleInit { } // --- TRANSACTION HANDLING --- // - @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 3600, process: Process.BANK_TX }) + @DfxCron(CronExpression.EVERY_30_SECONDS, { scope: CronScope.Worker, timeout: 3600, process: Process.BANK_TX }) async checkBankTx(): Promise { try { await this.checkTransactions(); @@ -148,7 +148,7 @@ export class BankTxService implements OnModuleInit { await this.fillBankTx(); } - @DfxCron(CronExpression.EVERY_5_MINUTES, { process: Process.BANK_TX }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.Worker, process: Process.BANK_TX }) async enrichYapealTransactions(): Promise { const transactions = await this.bankTxRepo.find({ where: { familyCode: 'CCRD' }, // credit card => wrong data diff --git a/src/subdomains/supporting/bank/bank-account/bank-account.service.ts b/src/subdomains/supporting/bank/bank-account/bank-account.service.ts index 7a7cbce1bb..c994e58ee4 100644 --- a/src/subdomains/supporting/bank/bank-account/bank-account.service.ts +++ b/src/subdomains/supporting/bank/bank-account/bank-account.service.ts @@ -3,7 +3,7 @@ import { CronExpression } from '@nestjs/schedule'; import { BankDetailsDto, IbanDetailsDto, IbanService } from 'src/integration/bank/services/iban.service'; import { CountryService } from 'src/shared/models/country/country.service'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { KycType } from 'src/subdomains/generic/user/models/user-data/user-data.enum'; import { Equal, IsNull, Like, Not } from 'typeorm'; import { BankAccount, BankAccountInfos } from './bank-account.entity'; @@ -35,7 +35,7 @@ export class BankAccountService { // --- INTERNAL METHODS --- // - @DfxCron(CronExpression.EVERY_WEEK, { process: Process.BANK_ACCOUNT, timeout: 3600 }) + @DfxCron(CronExpression.EVERY_WEEK, { scope: CronScope.Worker, process: Process.BANK_ACCOUNT, timeout: 3600 }) async checkFailedBankAccounts(): Promise { const failedBankAccounts = await this.bankAccountRepo.findBy({ returnCode: 256 }); for (const bankAccount of failedBankAccounts) { @@ -43,7 +43,7 @@ export class BankAccountService { } } - @DfxCron(CronExpression.EVERY_HOUR, { process: Process.BANK_ACCOUNT, timeout: 3600 }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, process: Process.BANK_ACCOUNT, timeout: 3600 }) async reloadErrorBankAccounts(): Promise { const bankAccounts = await this.bankAccountRepo.findBy({ result: Like('Error:%') }); for (const bankAccount of bankAccounts) { @@ -51,7 +51,7 @@ export class BankAccountService { } } - @DfxCron(CronExpression.EVERY_10_MINUTES, { process: Process.BANK_ACCOUNT, timeout: 3600 }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.BANK_ACCOUNT, timeout: 3600 }) async reloadUncheckedBankAccounts(): Promise { const bankAccounts = await this.bankAccountRepo.findBy({ result: IsNull(), iban: Not(IsNull()) }); for (const bankAccount of bankAccounts) { diff --git a/src/subdomains/supporting/bank/virtual-iban/virtual-iban-frick-issuance-reconciliation.service.ts b/src/subdomains/supporting/bank/virtual-iban/virtual-iban-frick-issuance-reconciliation.service.ts index 84ee88b35e..72f3eb44ef 100644 --- a/src/subdomains/supporting/bank/virtual-iban/virtual-iban-frick-issuance-reconciliation.service.ts +++ b/src/subdomains/supporting/bank/virtual-iban/virtual-iban-frick-issuance-reconciliation.service.ts @@ -4,7 +4,7 @@ import { FrickVirtualIban, FrickVirtualIbanState } from 'src/integration/bank/dt import { FrickVirtualIbansFetchResult } from 'src/integration/bank/services/frick.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { IbanBankName } from 'src/subdomains/supporting/bank/bank/dto/bank.dto'; import { FrickVibanProvider } from 'src/subdomains/supporting/bank/virtual-iban/providers/frick-viban.provider'; import { VirtualIbanIssuanceEvent } from 'src/subdomains/supporting/bank/virtual-iban/virtual-iban-issuance-event.entity'; @@ -88,6 +88,7 @@ export class VirtualIbanFrickIssuanceReconciliationService { * and external cleanup targets exact vIBAN identities, making repeated work fail closed or idempotent. */ @DfxCron(CronExpression.EVERY_HOUR, { + scope: CronScope.Worker, process: Process.VIRTUAL_IBAN_FRICK_ISSUANCE_RECONCILIATION, timeout: 1800, }) diff --git a/src/subdomains/supporting/dex/services/dex.service.ts b/src/subdomains/supporting/dex/services/dex.service.ts index 168314cdfb..d0f571bb30 100644 --- a/src/subdomains/supporting/dex/services/dex.service.ts +++ b/src/subdomains/supporting/dex/services/dex.service.ts @@ -4,7 +4,7 @@ import { FeeAmount } from '@uniswap/v3-sdk'; import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.enum'; import { Asset } from 'src/shared/models/asset/asset.entity'; import { DfxLogger, LogLevel } from 'src/shared/services/dfx-logger'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { MailContext, MailType } from 'src/subdomains/supporting/notification/enums'; import { NotificationService } from 'src/subdomains/supporting/notification/services/notification.service'; @@ -322,7 +322,7 @@ export class DexService { } //*** JOBS ***// - @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800 }) + @DfxCron(CronExpression.EVERY_30_SECONDS, { scope: CronScope.Worker, timeout: 1800 }) async finalizePurchaseOrders(): Promise { await this.alertStrandedPurchaseOrders(); diff --git a/src/subdomains/supporting/fiat-output/fiat-output-frick.service.ts b/src/subdomains/supporting/fiat-output/fiat-output-frick.service.ts index cf267260cd..01ca34e7f4 100644 --- a/src/subdomains/supporting/fiat-output/fiat-output-frick.service.ts +++ b/src/subdomains/supporting/fiat-output/fiat-output-frick.service.ts @@ -12,7 +12,7 @@ import { BankFrickService } from 'src/integration/bank/services/frick.service'; import { IbanService } from 'src/integration/bank/services/iban.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { DisabledProcess, Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { FindOptionsWhere, In, IsNull, Not } from 'typeorm'; import { IbanBankName } from '../bank/bank/dto/bank.dto'; import { FiatOutput, TransactionCharge } from './fiat-output.entity'; @@ -28,7 +28,7 @@ export class FiatOutputFrickService { private readonly ibanService: IbanService, ) {} - @DfxCron(CronExpression.EVERY_HOUR, { process: Process.FIAT_OUTPUT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, process: Process.FIAT_OUTPUT, timeout: 1800 }) async checkFrickOrderStatus(): Promise { if (DisabledProcess(Process.FIAT_OUTPUT_FRICK_STATUS_CHECK)) return; if (!this.frickService.isAvailable()) return; diff --git a/src/subdomains/supporting/fiat-output/fiat-output-job.service.ts b/src/subdomains/supporting/fiat-output/fiat-output-job.service.ts index dbdc2db21c..3c5f655011 100644 --- a/src/subdomains/supporting/fiat-output/fiat-output-job.service.ts +++ b/src/subdomains/supporting/fiat-output/fiat-output-job.service.ts @@ -15,7 +15,7 @@ import { Country } from 'src/shared/models/country/country.entity'; import { CountryService } from 'src/shared/models/country/country.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { DisabledProcess, Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { FindOptionsWhere, In, IsNull, Like, Not } from 'typeorm'; import { BankTxRepeatService } from '../bank-tx/bank-tx-repeat/bank-tx-repeat.service'; @@ -64,7 +64,7 @@ export class FiatOutputJobService { private readonly bankService: BankService, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.FIAT_OUTPUT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.FIAT_OUTPUT, timeout: 1800 }) async fillFiatOutput() { await this.assignBankAccount(); await this.setReadyDate(); @@ -77,7 +77,7 @@ export class FiatOutputJobService { await this.notifyScryptDeposits(); } - @DfxCron(CronExpression.EVERY_HOUR, { process: Process.FIAT_OUTPUT }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, process: Process.FIAT_OUTPUT }) async checkOlkypayOrderStatus(): Promise { if (DisabledProcess(Process.FIAT_OUTPUT_OLKYPAY_STATUS_CHECK)) return; if (!this.olkypayService.isAvailable()) return; @@ -103,7 +103,7 @@ export class FiatOutputJobService { } } - @DfxCron(CronExpression.EVERY_HOUR, { process: Process.FIAT_OUTPUT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, process: Process.FIAT_OUTPUT, timeout: 1800 }) async generateReports() { const entities = await this.fiatOutputRepo.find({ where: { reportCreated: false, isComplete: true }, diff --git a/src/subdomains/supporting/fiat-payin/services/fiat-payin-sync.service.ts b/src/subdomains/supporting/fiat-payin/services/fiat-payin-sync.service.ts index c56e71ed63..a4b3897f8b 100644 --- a/src/subdomains/supporting/fiat-payin/services/fiat-payin-sync.service.ts +++ b/src/subdomains/supporting/fiat-payin/services/fiat-payin-sync.service.ts @@ -6,7 +6,7 @@ import { ChargebackReason, ChargebackState, TransactionStatus } from 'src/integr import { SiftService } from 'src/integration/sift/services/sift.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { BuyService } from 'src/subdomains/core/buy-crypto/routes/buy/buy.service'; import { TransactionSourceType } from '../../payment/entities/transaction.entity'; @@ -32,7 +32,7 @@ export class FiatPayInSyncService { // --- JOBS --- // - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.FIAT_PAY_IN, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.FIAT_PAY_IN, timeout: 1800 }) async syncCheckout() { if (!this.checkoutService.isAvailable()) { if (!this.unavailableWarningLogged) { diff --git a/src/subdomains/supporting/log/log-job.service.ts b/src/subdomains/supporting/log/log-job.service.ts index f85f7037b0..dab36d7c99 100644 --- a/src/subdomains/supporting/log/log-job.service.ts +++ b/src/subdomains/supporting/log/log-job.service.ts @@ -15,7 +15,7 @@ import { SettingService } from 'src/shared/models/setting/setting.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process, ProcessService } from 'src/shared/services/process.service'; import { AsyncCache, CacheItemResetPeriod } from 'src/shared/utils/async-cache'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { AmountType, Util } from 'src/shared/utils/util'; import { BuyCrypto } from 'src/subdomains/core/buy-crypto/process/entities/buy-crypto.entity'; import { BuyCryptoService } from 'src/subdomains/core/buy-crypto/process/services/buy-crypto.service'; @@ -114,7 +114,7 @@ export class LogJobService { private readonly dashboardFinancialService: DashboardFinancialService, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.TRADING_LOG, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.TRADING_LOG, timeout: 1800 }) async saveTradingLog() { try { // trading log diff --git a/src/subdomains/supporting/log/log.service.ts b/src/subdomains/supporting/log/log.service.ts index de207e4126..fbfc4c680b 100644 --- a/src/subdomains/supporting/log/log.service.ts +++ b/src/subdomains/supporting/log/log.service.ts @@ -3,7 +3,7 @@ import { CronExpression } from '@nestjs/schedule'; import { SettingService } from 'src/shared/models/setting/setting.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { CreateLogDto, LogCleanupSetting, UpdateLogDto } from './dto/create-log.dto'; import { SetFinancialLogValidityDto } from './dto/set-financial-log-validity.dto'; import { @@ -24,7 +24,7 @@ export class LogService { private readonly settingService: SettingService, ) {} - @DfxCron(CronExpression.EVERY_DAY_AT_11PM, { process: Process.LOG_CLEANUP }) + @DfxCron(CronExpression.EVERY_DAY_AT_11PM, { scope: CronScope.Worker, process: Process.LOG_CLEANUP }) async cleanup(): Promise { const logCleanupSettings = await this.settingService.getObj('logCleanup', []); diff --git a/src/subdomains/supporting/notification/services/notification-job.service.ts b/src/subdomains/supporting/notification/services/notification-job.service.ts index 42b3db3442..c4fdf47167 100644 --- a/src/subdomains/supporting/notification/services/notification-job.service.ts +++ b/src/subdomains/supporting/notification/services/notification-job.service.ts @@ -3,7 +3,7 @@ import { Injectable } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { LessThanOrEqual } from 'typeorm'; import { MailFactory } from '../factories/mail.factory'; @@ -33,7 +33,7 @@ export class NotificationJobService { private readonly mailService: MailService, ) {} - @DfxCron(CronExpression.EVERY_10_MINUTES, { process: Process.MAIL_RETRY, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.MAIL_RETRY, timeout: 7200 }) async resendUncompletedMails(): Promise { const uncompletedMails = await this.notificationRepo.find({ where: { isComplete: false, created: LessThanOrEqual(Util.minutesBefore(1)) }, diff --git a/src/subdomains/supporting/payin/services/payin-notification.service.ts b/src/subdomains/supporting/payin/services/payin-notification.service.ts index b2654afc10..63202e090b 100644 --- a/src/subdomains/supporting/payin/services/payin-notification.service.ts +++ b/src/subdomains/supporting/payin/services/payin-notification.service.ts @@ -2,7 +2,7 @@ import { Injectable } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { MailContext, MailType } from 'src/subdomains/supporting/notification/enums'; import { MailFactory, @@ -23,7 +23,7 @@ export class PayInNotificationService { private readonly notificationService: NotificationService, ) {} - @DfxCron(CronExpression.EVERY_5_MINUTES, { process: Process.PAY_IN_MAIL, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.Worker, process: Process.PAY_IN_MAIL, timeout: 1800 }) async sendNotificationMails(): Promise { await this.returnedCryptoInput(); } diff --git a/src/subdomains/supporting/payin/services/payin.service.ts b/src/subdomains/supporting/payin/services/payin.service.ts index 0e3be09c49..223ede1b2b 100644 --- a/src/subdomains/supporting/payin/services/payin.service.ts +++ b/src/subdomains/supporting/payin/services/payin.service.ts @@ -6,7 +6,7 @@ import { Asset } from 'src/shared/models/asset/asset.entity'; import { BlockchainAddress } from 'src/shared/models/blockchain-address'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { CheckStatus } from 'src/subdomains/core/aml/enums/check-status.enum'; import { Swap } from 'src/subdomains/core/buy-crypto/routes/swap/swap.entity'; @@ -326,26 +326,26 @@ export class PayInService { // --- JOBS --- // - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.PAY_IN, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.PAY_IN, timeout: 7200 }) async forwardPayInEntries(): Promise { await this.forwardPayIns(); await this.processStrandedSendingPayIns(); } - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.PAY_IN, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.PAY_IN, timeout: 7200 }) async returnPayInEntries(): Promise { await this.returnPayIns(); await this.processStrandedSendingPayIns(); } - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.PAY_IN, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.PAY_IN, timeout: 7200 }) async checkConfirmations(): Promise { await this.checkInputConfirmations(); await this.checkOutputConfirmations(); await this.checkReturnConfirmations(); } - @DfxCron(CronExpression.EVERY_10_MINUTES, { process: Process.PAY_IN, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.PAY_IN, timeout: 7200 }) async updateFailedPayments(): Promise { const checkDate = Util.minutesBefore(15); diff --git a/src/subdomains/supporting/payin/strategies/register/impl/base/citrea.strategy.ts b/src/subdomains/supporting/payin/strategies/register/impl/base/citrea.strategy.ts index fb253de7d7..e1cfefe2e6 100644 --- a/src/subdomains/supporting/payin/strategies/register/impl/base/citrea.strategy.ts +++ b/src/subdomains/supporting/payin/strategies/register/impl/base/citrea.strategy.ts @@ -6,7 +6,7 @@ import { Asset, AssetType } from 'src/shared/models/asset/asset.entity'; import { BlockchainAddress } from 'src/shared/models/blockchain-address'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { TransactionRequestService } from 'src/subdomains/supporting/payment/services/transaction-request.service'; import { PayInType } from '../../../../entities/crypto-input.entity'; @@ -37,7 +37,7 @@ export abstract class CitreaBaseStrategy extends RegisterStrategy { } // --- JOBS --- // - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.PAY_IN, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.PAY_IN, timeout: 7200 }) async checkPayInEntries(): Promise { const activeDepositAddresses = await this.transactionRequestService.getActiveDepositAddresses( Util.hoursBefore(1), diff --git a/src/subdomains/supporting/payin/strategies/register/impl/bitcoin.strategy.ts b/src/subdomains/supporting/payin/strategies/register/impl/bitcoin.strategy.ts index e8c34daffa..b738facde7 100644 --- a/src/subdomains/supporting/payin/strategies/register/impl/bitcoin.strategy.ts +++ b/src/subdomains/supporting/payin/strategies/register/impl/bitcoin.strategy.ts @@ -6,7 +6,7 @@ import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.e import { BlockchainAddress } from 'src/shared/models/blockchain-address'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { DepositService } from 'src/subdomains/supporting/address-pool/deposit/deposit.service'; import { PayInType } from '../../../entities/crypto-input.entity'; @@ -31,7 +31,7 @@ export class BitcoinStrategy extends PollingStrategy { } //*** JOBS ***// - @DfxCron(CronExpression.EVERY_SECOND, { process: Process.PAY_IN, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_SECOND, { scope: CronScope.Worker, process: Process.PAY_IN, timeout: 7200 }) async checkPayInEntries(): Promise { if (!this.payInBitcoinService.isAvailable()) { if (!this.unavailableWarningLogged) { diff --git a/src/subdomains/supporting/payin/strategies/register/impl/cardano.strategy.ts b/src/subdomains/supporting/payin/strategies/register/impl/cardano.strategy.ts index a5f00c7f03..2bd4a9abee 100644 --- a/src/subdomains/supporting/payin/strategies/register/impl/cardano.strategy.ts +++ b/src/subdomains/supporting/payin/strategies/register/impl/cardano.strategy.ts @@ -8,7 +8,7 @@ import { Asset, AssetType } from 'src/shared/models/asset/asset.entity'; import { BlockchainAddress } from 'src/shared/models/blockchain-address'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { TransactionRequestService } from 'src/subdomains/supporting/payment/services/transaction-request.service'; import { PayInType } from '../../../entities/crypto-input.entity'; @@ -41,7 +41,7 @@ export class CardanoStrategy extends RegisterStrategy { } //*** JOBS ***// - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.PAY_IN, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.PAY_IN, timeout: 7200 }) async checkPayInEntries(): Promise { // not configured (no Tatum API key) -> skip, warn once if (!this.payInCardanoService.isConfigured) { diff --git a/src/subdomains/supporting/payin/strategies/register/impl/firo.strategy.ts b/src/subdomains/supporting/payin/strategies/register/impl/firo.strategy.ts index cfe6551888..8c9c545783 100644 --- a/src/subdomains/supporting/payin/strategies/register/impl/firo.strategy.ts +++ b/src/subdomains/supporting/payin/strategies/register/impl/firo.strategy.ts @@ -6,7 +6,7 @@ import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.e import { BlockchainAddress } from 'src/shared/models/blockchain-address'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { DepositService } from 'src/subdomains/supporting/address-pool/deposit/deposit.service'; import { PayInType } from '../../../entities/crypto-input.entity'; @@ -31,7 +31,7 @@ export class FiroStrategy extends PollingStrategy { } //*** JOBS ***// - @DfxCron(CronExpression.EVERY_SECOND, { process: Process.PAY_IN, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_SECOND, { scope: CronScope.Worker, process: Process.PAY_IN, timeout: 7200 }) async checkPayInEntries(): Promise { if (!this.payInFiroService.isAvailable()) { if (!this.unavailableWarningLogged) { diff --git a/src/subdomains/supporting/payin/strategies/register/impl/icp.strategy.ts b/src/subdomains/supporting/payin/strategies/register/impl/icp.strategy.ts index 7843ab5541..820e3ea606 100644 --- a/src/subdomains/supporting/payin/strategies/register/impl/icp.strategy.ts +++ b/src/subdomains/supporting/payin/strategies/register/impl/icp.strategy.ts @@ -8,7 +8,7 @@ import { BlockchainAddress } from 'src/shared/models/blockchain-address'; import { SettingService } from 'src/shared/models/setting/setting.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { DepositService } from 'src/subdomains/supporting/address-pool/deposit/deposit.service'; import { PayInType } from '../../../entities/crypto-input.entity'; import { PayInEntry } from '../../../interfaces'; @@ -40,7 +40,7 @@ export class InternetComputerStrategy extends RegisterStrategy { } //*** JOBS ***// - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.PAY_IN, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.PAY_IN, timeout: 7200 }) async checkPayInEntries(): Promise { const allDeposits = await this.depositService.getUsedDepositsByBlockchain(this.blockchain); const allDepositAddresses = allDeposits.map((d) => d.address); diff --git a/src/subdomains/supporting/payin/strategies/register/impl/monero.strategy.ts b/src/subdomains/supporting/payin/strategies/register/impl/monero.strategy.ts index dfbea80073..745fb6ff95 100644 --- a/src/subdomains/supporting/payin/strategies/register/impl/monero.strategy.ts +++ b/src/subdomains/supporting/payin/strategies/register/impl/monero.strategy.ts @@ -6,7 +6,7 @@ import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.e import { BlockchainAddress } from 'src/shared/models/blockchain-address'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { PayInType } from '../../../entities/crypto-input.entity'; import { PayInEntry } from '../../../interfaces'; @@ -26,7 +26,7 @@ export class MoneroStrategy extends PollingStrategy { } //*** JOBS ***// - @DfxCron(CronExpression.EVERY_SECOND, { process: Process.PAY_IN, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_SECOND, { scope: CronScope.Worker, process: Process.PAY_IN, timeout: 7200 }) async checkPayInEntries(): Promise { return super.checkPayInEntries(); } diff --git a/src/subdomains/supporting/payin/strategies/register/impl/zano.strategy.ts b/src/subdomains/supporting/payin/strategies/register/impl/zano.strategy.ts index 8db0d38816..e50eafce08 100644 --- a/src/subdomains/supporting/payin/strategies/register/impl/zano.strategy.ts +++ b/src/subdomains/supporting/payin/strategies/register/impl/zano.strategy.ts @@ -8,7 +8,7 @@ import { Asset } from 'src/shared/models/asset/asset.entity'; import { BlockchainAddress } from 'src/shared/models/blockchain-address'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { PayInType } from '../../../entities/crypto-input.entity'; import { PayInEntry } from '../../../interfaces'; @@ -28,7 +28,7 @@ export class ZanoStrategy extends PollingStrategy { } //*** JOBS ***// - @DfxCron(CronExpression.EVERY_SECOND, { process: Process.PAY_IN, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_SECOND, { scope: CronScope.Worker, process: Process.PAY_IN, timeout: 7200 }) async checkPayInEntries(): Promise { return super.checkPayInEntries(); } diff --git a/src/subdomains/supporting/payment/services/fee.service.ts b/src/subdomains/supporting/payment/services/fee.service.ts index 401b422333..c68ca84f5a 100644 --- a/src/subdomains/supporting/payment/services/fee.service.ts +++ b/src/subdomains/supporting/payment/services/fee.service.ts @@ -16,7 +16,7 @@ import { FiatService } from 'src/shared/models/fiat/fiat.service'; import { SettingService } from 'src/shared/models/setting/setting.service'; import { DfxLogger, LogLevel } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { AccountType } from 'src/subdomains/generic/user/models/user-data/account-type.enum'; import { UserData } from 'src/subdomains/generic/user/models/user-data/user-data.entity'; @@ -88,7 +88,11 @@ export class FeeService { ) {} // --- JOBS --- // - @DfxCron(CronExpression.EVERY_10_MINUTES, { process: Process.BLOCKCHAIN_FEE_UPDATE, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { + scope: CronScope.Worker, + process: Process.BLOCKCHAIN_FEE_UPDATE, + timeout: 1800, + }) async updateBlockchainFees() { const blockchainFees = await this.blockchainFeeRepo.find({ relations: { asset: true } }); diff --git a/src/subdomains/supporting/payment/services/transaction-notification.service.ts b/src/subdomains/supporting/payment/services/transaction-notification.service.ts index 9044b1312a..60c17f4509 100644 --- a/src/subdomains/supporting/payment/services/transaction-notification.service.ts +++ b/src/subdomains/supporting/payment/services/transaction-notification.service.ts @@ -2,7 +2,7 @@ import { forwardRef, Inject, Injectable } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { DisabledProcess, Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { BuyCrypto } from 'src/subdomains/core/buy-crypto/process/entities/buy-crypto.entity'; import { BuyFiat } from 'src/subdomains/core/sell-crypto/process/buy-fiat.entity'; @@ -27,7 +27,7 @@ export class TransactionNotificationService { private readonly bankTxService: BankTxService, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.TX_MAIL, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.TX_MAIL, timeout: 1800 }) async sendNotificationMails(): Promise { await this.txAssigned(); if (!DisabledProcess(Process.TX_UNASSIGNED_MAIL)) await this.txUnassigned(); diff --git a/src/subdomains/supporting/payment/services/transaction-request.service.ts b/src/subdomains/supporting/payment/services/transaction-request.service.ts index 41cc114058..a42906e0f6 100644 --- a/src/subdomains/supporting/payment/services/transaction-request.service.ts +++ b/src/subdomains/supporting/payment/services/transaction-request.service.ts @@ -7,7 +7,7 @@ import { AssetService } from 'src/shared/models/asset/asset.service'; import { FiatService } from 'src/shared/models/fiat/fiat.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { BuyService } from 'src/subdomains/core/buy-crypto/routes/buy/buy.service'; import { BuyPaymentInfoDto } from 'src/subdomains/core/buy-crypto/routes/buy/dto/buy-payment-info.dto'; @@ -49,13 +49,17 @@ export class TransactionRequestService { private readonly swapService: SwapService, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.TX_REQUEST, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.TX_REQUEST, timeout: 7200 }) async txRequestStatusSync() { await this.syncStatus(); await this.deleteOldTxRequests(); } - @DfxCron(CronExpression.EVERY_DAY_AT_3AM, { process: Process.TX_REQUEST_WAITING_EXPIRY, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_DAY_AT_3AM, { + scope: CronScope.Worker, + process: Process.TX_REQUEST_WAITING_EXPIRY, + timeout: 7200, + }) async txRequestWaitingExpiryCheck() { const expiryDate = Util.daysBefore(Config.txRequestWaitingExpiryDays); const entities = await this.transactionRequestRepo.findBy({ diff --git a/src/subdomains/supporting/payout/services/payout.service.ts b/src/subdomains/supporting/payout/services/payout.service.ts index c078f8ef3a..d49017d9a7 100644 --- a/src/subdomains/supporting/payout/services/payout.service.ts +++ b/src/subdomains/supporting/payout/services/payout.service.ts @@ -3,7 +3,7 @@ import { CronExpression } from '@nestjs/schedule'; import { Asset } from 'src/shared/models/asset/asset.entity'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { DisabledProcess, Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { MailContext, MailType } from 'src/subdomains/supporting/notification/enums'; import { NotificationService } from 'src/subdomains/supporting/notification/services/notification.service'; @@ -173,7 +173,7 @@ export class PayoutService { } //*** JOBS ***// - @DfxCron(CronExpression.EVERY_30_SECONDS, { process: Process.PAY_OUT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_30_SECONDS, { scope: CronScope.Worker, process: Process.PAY_OUT, timeout: 1800 }) async processOrders(): Promise { await this.checkExistingOrders(); await this.prepareNewOrders(); diff --git a/src/subdomains/supporting/pricing/services/asset-prices-job.service.ts b/src/subdomains/supporting/pricing/services/asset-prices-job.service.ts index c6ca2ed6b6..c61a94fdbd 100644 --- a/src/subdomains/supporting/pricing/services/asset-prices-job.service.ts +++ b/src/subdomains/supporting/pricing/services/asset-prices-job.service.ts @@ -6,7 +6,7 @@ import { UpdateResult } from 'src/shared/models/entity'; import { FiatService } from 'src/shared/models/fiat/fiat.service'; import { DfxLogger, LogLevel } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { MoreThanOrEqual } from 'typeorm'; import { PriceInvalidException } from '../domain/exceptions/price-invalid.exception'; @@ -24,7 +24,7 @@ export class AssetPricesJobService { private readonly assetPriceRepo: AssetPriceRepository, ) {} - @DfxCron(CronExpression.EVERY_HOUR, { process: Process.PRICING, timeout: 3600 }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, process: Process.PRICING, timeout: 3600 }) async updatePrices() { const assetsToUpdate = await this.assetService.getPricedAssets(); const updates: UpdateResult[] = []; @@ -59,7 +59,7 @@ export class AssetPricesJobService { await this.assetService.updateAssets(updates); } - @DfxCron(CronExpression.EVERY_5_MINUTES, { process: Process.PRICING, timeout: 3600 }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.Worker, process: Process.PRICING, timeout: 3600 }) async updatePaymentPrices() { const relevantFiats = await this.fiatService.getActiveFiat(); const relevantAssets = await this.assetService.getPaymentAssets(); diff --git a/src/subdomains/supporting/pricing/services/fiat-prices.service.ts b/src/subdomains/supporting/pricing/services/fiat-prices.service.ts index 102c52c712..3cf424739a 100644 --- a/src/subdomains/supporting/pricing/services/fiat-prices.service.ts +++ b/src/subdomains/supporting/pricing/services/fiat-prices.service.ts @@ -3,7 +3,7 @@ import { CronExpression } from '@nestjs/schedule'; import { FiatService } from 'src/shared/models/fiat/fiat.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { PriceCurrency, PriceValidity, PricingService } from './pricing.service'; @Injectable() @@ -16,7 +16,7 @@ export class FiatPricesService { ) {} // --- JOBS --- // - @DfxCron(CronExpression.EVERY_HOUR, { process: Process.PRICING, timeout: 3600 }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, process: Process.PRICING, timeout: 3600 }) async updatePrices() { const fiats = await this.fiatService.getActiveFiat(); diff --git a/src/subdomains/supporting/realunit/realunit-job.service.ts b/src/subdomains/supporting/realunit/realunit-job.service.ts index a16ea34ee3..1842df939e 100644 --- a/src/subdomains/supporting/realunit/realunit-job.service.ts +++ b/src/subdomains/supporting/realunit/realunit-job.service.ts @@ -3,7 +3,7 @@ import { CronExpression } from '@nestjs/schedule'; import { Config } from 'src/config/config'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { TransactionRequestService } from 'src/subdomains/supporting/payment/services/transaction-request.service'; import { HistoryEventDto } from './dto/realunit.dto'; @@ -21,7 +21,11 @@ export class RealUnitJobService { // Completes open REALU buy quotes as soon as the shares arrive on-chain. Share allocations // triggered outside the DFX payment flow (e.g. booked manually by the issuer) would otherwise // leave the quote in WaitingForPayment and keep showing a pending payment to the customer. - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.REALUNIT_QUOTE_COMPLETION, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { + scope: CronScope.Worker, + process: Process.REALUNIT_QUOTE_COMPLETION, + timeout: 1800, + }) async completeSettledQuotes(): Promise { const realuAsset = await this.realunitService.getRealuAsset(); const openQuotes = await this.transactionRequestService.getOpenBuyQuotes(realuAsset.id); @@ -87,7 +91,11 @@ export class RealUnitJobService { // Resolves RealUnit W2W transfer requests stuck in PROCESSING after a crash/restart between the // atomic claim and the broadcast/callback in confirmTransfer — see // RealUnitService.reconcilePendingTransfers for the actual reconciliation logic. - @DfxCron(CronExpression.EVERY_5_MINUTES, { process: Process.REALUNIT_TRANSFER_RECONCILIATION, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { + scope: CronScope.Worker, + process: Process.REALUNIT_TRANSFER_RECONCILIATION, + timeout: 1800, + }) async reconcilePendingTransfers(): Promise { await this.realunitService.reconcilePendingTransfers(); } diff --git a/src/subdomains/supporting/support-issue/services/limit-request-notification.service.ts b/src/subdomains/supporting/support-issue/services/limit-request-notification.service.ts index 6653c61dab..74d1719aa3 100644 --- a/src/subdomains/supporting/support-issue/services/limit-request-notification.service.ts +++ b/src/subdomains/supporting/support-issue/services/limit-request-notification.service.ts @@ -3,7 +3,7 @@ import { CronExpression } from '@nestjs/schedule'; import { Config } from 'src/config/config'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { MailContext, MailType } from 'src/subdomains/supporting/notification/enums'; import { MailKey, MailTranslationKey } from 'src/subdomains/supporting/notification/factories/mail.factory'; @@ -22,7 +22,11 @@ export class LimitRequestNotificationService { private readonly notificationService: NotificationService, ) {} - @DfxCron(CronExpression.EVERY_5_MINUTES, { process: Process.LIMIT_REQUEST_MAIL, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { + scope: CronScope.Worker, + process: Process.LIMIT_REQUEST_MAIL, + timeout: 1800, + }) async sendNotificationMails(): Promise { await this.limitRequestAcceptedManual(); } diff --git a/src/subdomains/supporting/support-issue/services/support-escalation.service.ts b/src/subdomains/supporting/support-issue/services/support-escalation.service.ts index cfc8855cb5..6fb6d6c89a 100644 --- a/src/subdomains/supporting/support-issue/services/support-escalation.service.ts +++ b/src/subdomains/supporting/support-issue/services/support-escalation.service.ts @@ -5,7 +5,7 @@ import { SettingService } from 'src/shared/models/setting/setting.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { HttpService } from 'src/shared/services/http.service'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { In } from 'typeorm'; import { SupportIssueReasonLabelMap, SupportIssueTypeLabelMap } from '../dto/support-issue-label'; @@ -200,7 +200,7 @@ export class SupportEscalationService { // --- Escalation detection --- - @DfxCron(CronExpression.EVERY_5_MINUTES, { process: Process.SUPPORT_BOT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.Worker, process: Process.SUPPORT_BOT, timeout: 1800 }) async checkEscalations(): Promise { if (!this.token) return; diff --git a/src/subdomains/supporting/support-issue/services/support-issue-job.service.ts b/src/subdomains/supporting/support-issue/services/support-issue-job.service.ts index a9fd636bb3..39ec3ae8f2 100644 --- a/src/subdomains/supporting/support-issue/services/support-issue-job.service.ts +++ b/src/subdomains/supporting/support-issue/services/support-issue-job.service.ts @@ -3,7 +3,7 @@ import { CronExpression } from '@nestjs/schedule'; import { Config } from 'src/config/config'; import { SettingService } from 'src/shared/models/setting/setting.service'; import { Process } from 'src/shared/services/process.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { CheckStatus } from 'src/subdomains/core/aml/enums/check-status.enum'; import { BuyCryptoStatus } from 'src/subdomains/core/buy-crypto/process/entities/buy-crypto.entity'; @@ -32,7 +32,7 @@ export class SupportIssueJobService { private readonly settingsService: SettingService, ) {} - @DfxCron(CronExpression.EVERY_HOUR, { process: Process.SUPPORT_BOT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, process: Process.SUPPORT_BOT, timeout: 1800 }) async autoOnHold() { const entities = await this.supportIssueRepo.find({ where: { @@ -52,7 +52,7 @@ export class SupportIssueJobService { } } - @DfxCron(CronExpression.EVERY_MINUTE, { process: Process.SUPPORT_BOT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.SUPPORT_BOT, timeout: 1800 }) async sendAutoResponses() { const disabledTemplates = await this.settingsService .get('supportBot') From ac041b0ba3d285c4a9433d02f8b3a3d6ff5e51c7 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:24 +0200 Subject: [PATCH 12/86] Make three jobs individually switchable The runtime kill switch works per Process, and a job declaring none cannot be stopped without a deploy. That is the tool the split relies on when a single job misbehaves in the process it was moved to, so the three jobs where stopping one matters get a flag: - DexService::finalizePurchaseOrders writes liquidity orders, the only one of the three carrying real financial risk. - RefService::checkRefs removes rows. - JwtRevocationSyncService::syncDeniedJwtAccounts writes a setting. It is idempotent, but it should run once rather than not at all. The remaining jobs without a flag are the monthly and annual volume resets. Their statement finds nothing left to do on a second run, so a flag would be good practice but is not a precondition for anything here. --- src/shared/services/process.service.ts | 3 +++ src/subdomains/core/referral/process/ref.service.ts | 3 ++- .../user/models/user-data/jwt-revocation-sync.service.ts | 7 ++++++- src/subdomains/supporting/dex/services/dex.service.ts | 7 ++++++- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/shared/services/process.service.ts b/src/shared/services/process.service.ts index e3caca5bae..1395ecccf2 100644 --- a/src/shared/services/process.service.ts +++ b/src/shared/services/process.service.ts @@ -118,6 +118,9 @@ export enum Process { LEDGER_MARK_TO_MARKET = 'LedgerMarkToMarket', LEDGER_CUTOVER = 'LedgerCutover', LEDGER_COA_BOOTSTRAP = 'LedgerCoaBootstrap', + DEX_PURCHASE_ORDER = 'DexPurchaseOrder', + REF_CLEANUP = 'RefCleanup', + JWT_REVOCATION_SYNC = 'JwtRevocationSync', } const safetyProcesses: Process[] = [ diff --git a/src/subdomains/core/referral/process/ref.service.ts b/src/subdomains/core/referral/process/ref.service.ts index 87fc7edc61..3af692a81d 100644 --- a/src/subdomains/core/referral/process/ref.service.ts +++ b/src/subdomains/core/referral/process/ref.service.ts @@ -1,6 +1,7 @@ import { Injectable } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; import { DfxLogger } from 'src/shared/services/dfx-logger'; +import { Process } from 'src/shared/services/process.service'; import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { IsNull, LessThan } from 'typeorm'; @@ -15,7 +16,7 @@ export class RefService { constructor(private readonly repo: RefRepository) {} - @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, process: Process.REF_CLEANUP, timeout: 7200 }) async checkRefs(): Promise { const expirationDate = Util.daysBefore(this.refExpirationDays); diff --git a/src/subdomains/generic/user/models/user-data/jwt-revocation-sync.service.ts b/src/subdomains/generic/user/models/user-data/jwt-revocation-sync.service.ts index 9cc38c1c89..21813a0f15 100644 --- a/src/subdomains/generic/user/models/user-data/jwt-revocation-sync.service.ts +++ b/src/subdomains/generic/user/models/user-data/jwt-revocation-sync.service.ts @@ -1,6 +1,7 @@ import { Injectable } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; import { SettingService } from 'src/shared/models/setting/setting.service'; +import { Process } from 'src/shared/services/process.service'; import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { In } from 'typeorm'; import { RiskStatus, UserDataStatus } from './user-data.enum'; @@ -25,7 +26,11 @@ export class JwtRevocationSyncService { // Runs every minute: fast revocation of a blocked or compromised account is a security requirement that // warrants the security-revocation exception to the "prefer 15min" cron guideline. - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { + scope: CronScope.Worker, + process: Process.JWT_REVOCATION_SYNC, + timeout: 1800, + }) async syncDeniedJwtAccounts(): Promise { const blockedAccounts = await this.userDataRepo.find({ select: { id: true }, diff --git a/src/subdomains/supporting/dex/services/dex.service.ts b/src/subdomains/supporting/dex/services/dex.service.ts index d0f571bb30..1b18435239 100644 --- a/src/subdomains/supporting/dex/services/dex.service.ts +++ b/src/subdomains/supporting/dex/services/dex.service.ts @@ -4,6 +4,7 @@ import { FeeAmount } from '@uniswap/v3-sdk'; import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.enum'; import { Asset } from 'src/shared/models/asset/asset.entity'; import { DfxLogger, LogLevel } from 'src/shared/services/dfx-logger'; +import { Process } from 'src/shared/services/process.service'; import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { MailContext, MailType } from 'src/subdomains/supporting/notification/enums'; @@ -322,7 +323,11 @@ export class DexService { } //*** JOBS ***// - @DfxCron(CronExpression.EVERY_30_SECONDS, { scope: CronScope.Worker, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_30_SECONDS, { + scope: CronScope.Worker, + process: Process.DEX_PURCHASE_ORDER, + timeout: 1800, + }) async finalizePurchaseOrders(): Promise { await this.alertStrandedPurchaseOrders(); From d281a02a556a34bde17d54e9e2161d627e4c7aea Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:25 +0200 Subject: [PATCH 13/86] Keep periodic work from being registered outside the scheduler Two native @Cron decorators and one bare setInterval had grown alongside DfxCronService and are invisible to it. A scope cannot reach them, so after the split they would run in both processes: the transaction request sync deletes rows, and the Spark timer performs on-chain wallet maintenance against a single seed. Nothing in the repository would have flagged the next one. Both decorators had already moved to @DfxCron; txRequestStatusSync now also carries useDelay: false. DfxCron staggers job starts by default, and for a minute-based expression that spreads them over up to 30 seconds - without the flag the move would have changed when the job runs, not just how it is registered. The daily job reaches no branch of the delay logic and is unaffected. The guard test is deliberately syntactic: it asks whether @Cron( or setInterval( occurs, not whether the code behind it is safe. Its exception list therefore has a natural ceiling, and the one entry - a timer tied to the lifetime of a client object rather than to a schedule - is a decision, not a special case. A third test asserts each exception still matches something, so a leftover entry cannot quietly read as a rule that still applies. --- .../__tests__/cron-registration.guard.spec.ts | 73 +++++++++++++++++++ .../services/transaction-request.service.ts | 10 ++- 2 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 src/shared/services/__tests__/cron-registration.guard.spec.ts diff --git a/src/shared/services/__tests__/cron-registration.guard.spec.ts b/src/shared/services/__tests__/cron-registration.guard.spec.ts new file mode 100644 index 0000000000..0b91b6c86b --- /dev/null +++ b/src/shared/services/__tests__/cron-registration.guard.spec.ts @@ -0,0 +1,73 @@ +import { readFileSync, readdirSync, statSync } from 'fs'; +import { join, relative } from 'path'; + +const SRC = join(__dirname, '..', '..', '..'); + +/** + * Periodic work registered outside DfxCronService is invisible to the scope mechanism: it runs + * in every process, which for anything writing to the database or driving business forward means + * running twice without a shared lock. Two native @Cron decorators and one bare setInterval had + * grown that way before the mechanism existed, and nothing would have flagged the next one. + * + * The check is syntactic on purpose. It asks whether a pattern occurs, not whether the code + * behind it is safe, so its exception list has a natural ceiling and every entry is a deliberate + * decision rather than a special case. + */ +const FORBIDDEN: { pattern: RegExp; what: string; instead: string }[] = [ + { + pattern: /@Cron\(/, + what: 'the native @Cron decorator', + instead: 'use @DfxCron, which applies the scope, the process flag and the lock', + }, + { + pattern: /\bsetInterval\(/, + what: 'a bare setInterval', + instead: 'use @DfxCron, or bind the timer to Config.cronRole where a scheduler cannot reach it', + }, +]; + +/** + * A timer tied to the lifetime of an object rather than to a schedule. It is bound to the role + * directly, which is the same decision the scope expresses for a cron job. + */ +const ALLOWED = ['integration/blockchain/spark/spark-client.ts']; + +function sourceFiles(dir: string): string[] { + return readdirSync(dir).flatMap((entry) => { + const path = join(dir, entry); + + if (statSync(path).isDirectory()) return entry === 'node_modules' ? [] : sourceFiles(path); + if (!entry.endsWith('.ts') || entry.endsWith('.spec.ts')) return []; + + return [path]; + }); +} + +describe('cron registration', () => { + const files = sourceFiles(SRC).map((path) => ({ + path: relative(SRC, path).split('\\').join('/'), + content: readFileSync(path, 'utf8'), + })); + + it('finds source files to check', () => { + // Guards against the check passing because the traversal returned nothing. + expect(files.length).toBeGreaterThan(100); + }); + + it.each(FORBIDDEN)('registers no periodic work through $what — $instead', ({ pattern }) => { + const offenders = files.filter((f) => !ALLOWED.includes(f.path) && pattern.test(f.content)).map((f) => f.path); + + expect(offenders).toEqual([]); + }); + + it('keeps the exception list honest', () => { + // An exception that no longer matches anything is a leftover, and the next reader would take + // it for a rule that still applies. + for (const allowed of ALLOWED) { + const file = files.find((f) => f.path === allowed); + + expect(file).toBeDefined(); + expect(FORBIDDEN.some((f) => f.pattern.test(file.content))).toBe(true); + } + }); +}); diff --git a/src/subdomains/supporting/payment/services/transaction-request.service.ts b/src/subdomains/supporting/payment/services/transaction-request.service.ts index a42906e0f6..5a981a5b15 100644 --- a/src/subdomains/supporting/payment/services/transaction-request.service.ts +++ b/src/subdomains/supporting/payment/services/transaction-request.service.ts @@ -49,7 +49,15 @@ export class TransactionRequestService { private readonly swapService: SwapService, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.TX_REQUEST, timeout: 7200 }) + // useDelay: false keeps the schedule this job had as a native @Cron. DfxCron staggers job + // starts by default, which for a minute-based expression spreads them over up to 30 seconds - + // moving it here would change when it runs, not just how it is registered. + @DfxCron(CronExpression.EVERY_MINUTE, { + scope: CronScope.Worker, + process: Process.TX_REQUEST, + useDelay: false, + timeout: 7200, + }) async txRequestStatusSync() { await this.syncStatus(); await this.deleteOldTxRequests(); From a9a3e23b2ae370635787ca7fd0b42f5d4d836804 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:27 +0200 Subject: [PATCH 14/86] Serve the monitoring state from where it is persisted The observers maintain their results in a BehaviorSubject inside the process running them, and eleven endpoints read that field: GET /monitoring/data and the six health paths. Scoping the observers to the worker would leave those answers frozen at the boot snapshot in the API process - the health report included - while scoping them to the API process would move AML, node and bank queries back into the request path, which is the work the split exists to move out. Neither is necessary, because the state is already persisted in full and reloaded at boot. Only the read path did not use it. It does now, with a 30-second process cache in front and one shared read for concurrent requests. The refresh sits before the branching: /monitoring/data takes subsystem and metric as query parameters, and refreshing only the unfiltered branch would leave exactly the filtered answers stale. What this process holds more recently still wins, so a single-process setup is as current as before and a value arriving through the webhook is visible before the next write. The write path needed its own answer. Every process subscribes to its own updates and writes the whole state as a single row, so whichever writes last would drop the other's work entirely - the API process overwriting the observers with its boot state, or the webhook value being overwritten by the next observer run. Only the metrics changed in this process are now merged into the stored row. Binding the write to one role would have been smaller, but it would leave the webhook path silently ineffective rather than working. The reload does not send mail on failure. At boot that notification runs once; on a path taken by every request it would answer a database outage with a flood of mail, during the outage. --- .../__tests__/monitoring.service.spec.ts | 126 ++++++++++++++++ .../core/monitoring/monitoring.service.ts | 141 ++++++++++++++---- 2 files changed, 241 insertions(+), 26 deletions(-) create mode 100644 src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts diff --git a/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts b/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts new file mode 100644 index 0000000000..af6ce546e0 --- /dev/null +++ b/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts @@ -0,0 +1,126 @@ +import { createMock } from '@golevelup/ts-jest'; +import { NotFoundException } from '@nestjs/common'; +import { NotificationService } from 'src/subdomains/supporting/notification/services/notification.service'; +import { MonitoringService } from '../monitoring.service'; +import { Metric, SystemState } from '../system-state-snapshot.entity'; + +function snapshot(state: SystemState): { id: number; data: string } { + return { id: 1, data: JSON.stringify(state) }; +} + +function metric(data: unknown, updated: string): Metric { + return { data, updated: new Date(updated) }; +} + +describe('MonitoringService', () => { + let repo: any; + let notificationService: NotificationService; + let service: MonitoringService; + + // Timestamps in the past, so a value written during the test is always the later one. + const persisted: SystemState = { + node: { health: metric({ up: true }, '2020-01-01T00:10:00Z') }, + bank: { balance: metric({ chf: 42 }, '2020-01-01T00:10:00Z') }, + }; + + // As it comes back out of the database: JSON carries `updated` as a string, which serialises + // identically in the response. + const asRead = (): SystemState => JSON.parse(JSON.stringify(persisted)); + + beforeEach(() => { + repo = { findOne: jest.fn().mockResolvedValue(snapshot(persisted)), save: jest.fn().mockResolvedValue(undefined) }; + notificationService = createMock(); + + service = new MonitoringService(repo, notificationService); + }); + + describe('reading the state', () => { + it('answers from the persisted state, not from the in-memory state', async () => { + // Without this the endpoints behind GET /health* and /monitoring/data would answer from the + // boot snapshot in any process not running the observers. + await expect(service.getState(undefined, undefined)).resolves.toEqual(asRead()); + }); + + it('refreshes the filtered queries too', async () => { + // GET /monitoring/data takes subsystem and metric as query parameters. Refreshing only the + // unfiltered branch would leave exactly those answers stale. + await expect(service.getState('bank', undefined)).resolves.toEqual(asRead().bank); + await expect(service.getState('bank', 'balance')).resolves.toEqual(asRead().bank.balance); + }); + + it('still reports an unknown subsystem or metric as not found', async () => { + await expect(service.getState('ledger', undefined)).rejects.toThrow(NotFoundException); + await expect(service.getState('bank', 'turnover')).rejects.toThrow(NotFoundException); + }); + + it('prefers whichever value carries the later timestamp', () => { + const older: SystemState = { bank: { balance: metric({ chf: 42 }, '2020-01-01T00:10:00Z') } }; + const newer: SystemState = { bank: { balance: metric({ chf: 43 }, '2020-01-01T00:20:00Z') } }; + + expect(service['mergeNewer'](older, newer).bank.balance.data).toEqual({ chf: 43 }); + expect(service['mergeNewer'](newer, older).bank.balance.data).toEqual({ chf: 43 }); + }); + + it('shows a value produced in this process before it is persisted', async () => { + // The webhook path writes into the in-memory state of whichever process receives the call, + // and the persisted row follows only after the debounce. Reading the database alone would + // answer with the older value in that window. + await service['updateSystemState']('bank', 'balance', { chf: 99 }); + + const state = (await service.getState(undefined, undefined)) as SystemState; + + expect(state.bank.balance.data).toEqual({ chf: 99 }); + expect(state.node.health.data).toEqual({ up: true }); + }); + + it('reads at most once per cache window', async () => { + await service.getState(undefined, undefined); + await service.getState(undefined, undefined); + + expect(repo.findOne).toHaveBeenCalledTimes(1); + }); + + it('does not send a mail when the read fails', async () => { + // Unlike the load at start-up this path runs on every request: a database problem would + // otherwise answer itself with a flood of mails, precisely during the outage. + repo.findOne.mockRejectedValue(new Error('database unavailable')); + + await expect(service.getState(undefined, undefined)).resolves.toEqual({}); + expect(notificationService.sendMail).not.toHaveBeenCalled(); + }); + }); + + describe('persisting the state', () => { + it('keeps metrics another process wrote', async () => { + // Every process subscribes to its own updates and writes the same single row. Replacing it + // with this process's view would drop the other's work - the API process would overwrite + // the observers' results with its boot state. + const prev: SystemState = { bank: { balance: metric({ chf: 42 }, '2020-01-01T00:10:00Z') } }; + const next: SystemState = { bank: { balance: metric({ chf: 43 }, '2020-01-01T00:20:00Z') } }; + + await service['persist'](prev, next); + + const written = JSON.parse(repo.save.mock.calls[0][0].data) as SystemState; + + expect(written.bank.balance.data).toEqual({ chf: 43 }); + expect(written.node.health.data).toEqual({ up: true }); + }); + + it('writes nothing when no metric changed', async () => { + await service['persist'](persisted, persisted); + + expect(repo.save).not.toHaveBeenCalled(); + }); + + it('writes a metric that did not exist before', async () => { + const next: SystemState = { ledger: { open: metric({ count: 3 }, '2020-01-01T00:20:00Z') } }; + + await service['persist']({}, next); + + const written = JSON.parse(repo.save.mock.calls[0][0].data) as SystemState; + + expect(written.ledger.open.data).toEqual({ count: 3 }); + expect(written.bank.balance.data).toEqual({ chf: 42 }); + }); + }); +}); diff --git a/src/subdomains/core/monitoring/monitoring.service.ts b/src/subdomains/core/monitoring/monitoring.service.ts index 244575ae61..97831ee554 100644 --- a/src/subdomains/core/monitoring/monitoring.service.ts +++ b/src/subdomains/core/monitoring/monitoring.service.ts @@ -12,10 +12,14 @@ type SubsystemObservers = Map>; @Injectable() export class MonitoringService implements OnModuleInit { + private static readonly stateCacheMs = 30 * 1000; + private readonly logger = new DfxLogger(MonitoringService); #$state: BehaviorSubject = new BehaviorSubject({}); #observers: Map = new Map(); + #storedState?: { state: SystemState; loaded: number }; + #pendingLoad?: Promise; constructor( private systemStateSnapshotRepo: SystemStateSnapshotRepository, @@ -29,29 +33,28 @@ export class MonitoringService implements OnModuleInit { // *** PUBLIC API *** // async getState(subsystem: string, metric: string): Promise { + // Reading the in-memory state alone would only be correct in the process running the + // observers. The state is already persisted in full, so every read path takes it from there, + // with a short process cache in front. The refresh happens before the branching, not inside + // one of the branches: a filtered query is the same read and would otherwise stay stale. + const state = await this.currentState(); + if (!subsystem && !metric) { - return this.#$state.value; + return state; } if (subsystem && !metric) { - return this.getSubsystemState(subsystem); + return this.getSubsystemState(state, subsystem); } if (subsystem && metric) { - return this.getMetric(subsystem, metric); + return this.getMetric(state, subsystem, metric); } } async loadState(): Promise { try { - const latestPersistedState = await this.systemStateSnapshotRepo.findOne({ where: {}, order: { id: 'DESC' } }); - - if (!latestPersistedState) { - this.logger.warn('No monitoring state found in the database'); - return null; - } - - return JSON.parse(latestPersistedState.data); + return await this.readState(); } catch (e) { this.logger.error('Failed to parse loaded system state, defaulting to empty state:', e); @@ -120,11 +123,30 @@ export class MonitoringService implements OnModuleInit { .subscribe(([prevState, newState]) => this.persist(prevState, newState)); } + /** + * Writes only the metrics this process changed, merged into the stored state. + * + * The whole system state lives in a single row, and every process subscribing to its own + * updates writes it. Replacing the row with this process's view would drop whatever another + * process wrote in the meantime - the API process would overwrite the observers' work with its + * boot state, and the webhook path the other way round. Merging the changed metrics into the + * stored row makes it irrelevant which process writes. + */ private async persist(prevState: SystemState, newState: SystemState) { try { - if (this.hasStateChanged(prevState, newState)) { - await this.systemStateSnapshotRepo.save({ id: 1, data: JSON.stringify(newState) }); + const changed = this.changedMetrics(prevState, newState); + if (!changed.length) return; + + const stored = (await this.readState()) ?? {}; + const merged = cloneDeep(stored); + + for (const [subsystem, metric] of changed) { + merged[subsystem] = { ...(merged[subsystem] ?? {}), [metric]: newState[subsystem][metric] }; } + + await this.systemStateSnapshotRepo.save({ id: 1, data: JSON.stringify(merged) }); + + this.#storedState = { state: merged, loaded: Date.now() }; } catch (e) { this.logger.error('Error persisting the state:', e); @@ -136,22 +158,89 @@ export class MonitoringService implements OnModuleInit { } } - private hasStateChanged(prevState: SystemState, newState: SystemState): boolean { - if (!prevState && newState) return true; + /** The metrics whose data differs between the two states, as [subsystem, metric] pairs. */ + private changedMetrics(prevState: SystemState, newState: SystemState): [string, string][] { + return Object.entries(newState ?? {}).flatMap(([subsystemName, subsystemState]) => + Object.entries(subsystemState) + .filter( + ([metricName, newMetricState]) => + !isEqual(prevState?.[subsystemName]?.[metricName]?.data, newMetricState.data), + ) + .map(([metricName]) => [subsystemName, metricName] as [string, string]), + ); + } - return Object.entries(newState).some(([subsystemName, subsystemState]) => - Object.entries(subsystemState).some(([metricName, newMetricState]) => { - const prevMetricState = prevState[subsystemName] && prevState[subsystemName][metricName]; + /** Reads the persisted state without notifying: this runs on every read, not once at start. */ + private async readState(): Promise { + const latestPersistedState = await this.systemStateSnapshotRepo.findOne({ where: {}, order: { id: 'DESC' } }); - if (!prevMetricState && newMetricState) return true; + if (!latestPersistedState) { + this.logger.warn('No monitoring state found in the database'); + return null; + } - return !isEqual(prevMetricState.data, newMetricState.data); - }), - ); + return JSON.parse(latestPersistedState.data); + } + + /** + * The persisted state overlaid with anything this process holds more recently. Both matter: in + * a single-process setup the in-memory state is always the newer one, and a value arriving + * through the webhook is visible before the next persist. + */ + private async currentState(): Promise { + const stored = await this.storedState(); + + return stored ? this.mergeNewer(stored, this.#$state.value) : this.#$state.value; + } + + private async storedState(): Promise { + if (this.#storedState && Date.now() - this.#storedState.loaded < MonitoringService.stateCacheMs) { + return this.#storedState.state; + } + + // Concurrent requests share one read rather than each issuing their own. + if (!this.#pendingLoad) { + const load = this.readState().catch((e) => { + // Deliberately no mail: unlike the load at start-up, this path runs on every request, so + // a database problem would answer itself with a flood of mails. + this.logger.error('Failed to read the persisted system state:', e); + return null; + }); + + this.#pendingLoad = load; + void load.finally(() => { + if (this.#pendingLoad === load) this.#pendingLoad = undefined; + }); + } + + const state = await this.#pendingLoad; + if (state) this.#storedState = { state, loaded: Date.now() }; + + return state; + } + + /** Per metric, whichever of the two carries the later `updated` timestamp. */ + private mergeNewer(base: SystemState, overlay: SystemState): SystemState { + const merged = cloneDeep(base); + + for (const [subsystem, metrics] of Object.entries(overlay ?? {})) { + for (const [metric, state] of Object.entries(metrics)) { + if (this.updatedAt(state) >= this.updatedAt(merged[subsystem]?.[metric])) { + merged[subsystem] = { ...(merged[subsystem] ?? {}), [metric]: state }; + } + } + } + + return merged; + } + + /** Parsed JSON carries `updated` as a string, the in-memory state as a Date. */ + private updatedAt(metric?: Metric): number { + return metric?.updated ? new Date(metric.updated).getTime() : 0; } - private getSubsystemState(subsystem: string): SubsystemState { - const _subsystem = this.#$state.value[subsystem]; + private getSubsystemState(state: SystemState, subsystem: string): SubsystemState { + const _subsystem = state[subsystem]; if (!_subsystem) { throw new NotFoundException(`Subsystem not found, name: ${subsystem}`); @@ -159,8 +248,8 @@ export class MonitoringService implements OnModuleInit { return _subsystem; } - private getMetric(subsystem: string, metric: string): Metric { - const _subsystem = this.getSubsystemState(subsystem); + private getMetric(state: SystemState, subsystem: string, metric: string): Metric { + const _subsystem = this.getSubsystemState(state, subsystem); const _metric = _subsystem[metric]; if (!_metric) { From 50c9daab75cd575d070497a251884bdbc6ef5ddc Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:28 +0200 Subject: [PATCH 15/86] Keep live payment confirmation in the process serving the connections processExpiredPayments and checkTxConfirmations both reach PaymentLinkPaymentService.doSave(), which resolves the AsyncMap that PaymentLinkController's waitForPayment holds open and pushes the device activation into the RxJS subject the gateway delivers to its connected clients. Both are process-local, and both are read only from an HTTP path. The worker has no ingress, so no point-of-sale device ever connects to it. Run there, the confirmation would reach nobody: the waiting request hangs until the connection drops, because waitForPayment passes timeout 0 and AsyncMap.wait creates no timer for that, so the promise is never rejected. `Both` is no option: the jobs write to the database and trigger merchant webhooks, which two processes without a shared lock would do twice. The cost is deliberate - the confirmation work, blockchain client calls included, stays in the request process. Moving it would mean waking a waiting connection across process boundaries, and introducing that into the live payment path is a larger risk than leaving the work where it already runs. This is also why a binary cron switch could not express the split: some work is bound to the process holding the open connections. --- .../payment-link/services/payment-cron.service.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/subdomains/core/payment-link/services/payment-cron.service.ts b/src/subdomains/core/payment-link/services/payment-cron.service.ts index 6a2f20e8a3..f77886891f 100644 --- a/src/subdomains/core/payment-link/services/payment-cron.service.ts +++ b/src/subdomains/core/payment-link/services/payment-cron.service.ts @@ -16,14 +16,22 @@ export class PaymentCronService { private readonly paymentBalanceService: PaymentBalanceService, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.PAYMENT_EXPIRATION }) + // Api, not Worker: this job and checkTxConfirmations both end up in + // PaymentLinkPaymentService.doSave(), which resolves the AsyncMap that PaymentLinkController's + // waitForPayment is waiting on and pushes the device activation into the RxJS subject the + // gateway delivers to its connected clients. Both are confined to the process holding those + // connections, and the worker holds none. `Both` is no option either: the jobs write to the + // database and trigger merchant webhooks, which two processes without a shared lock would do + // twice. + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Api, process: Process.PAYMENT_EXPIRATION }) async processExpiredPayments(): Promise { await this.paymentLinkPaymentService.processExpiredPayments(); await this.paymentActivationService.processExpiredActivations(); await this.paymentQuoteService.processExpiredQuotes(); } - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.PAYMENT_CONFIRMATIONS }) + // Api for the same reason as processExpiredPayments above. + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Api, process: Process.PAYMENT_CONFIRMATIONS }) async checkTxConfirmations(): Promise { await this.paymentLinkPaymentService.checkTxConfirmations(); } From ae5e7ae26986236b59bec7624fe6c7b4da3a147f Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:29 +0200 Subject: [PATCH 16/86] Fill the dashboard balance store from a job in the API process LatestBalanceStore was written by LogJobService right after the financial aggregation, and GET /v1/dashboard/financial/latest reads it without ever touching the database - deliberately so, that endpoint used to parse a 43 kB message on every request and measured 23 ms median with a 1'989 ms p95. The aggregation belongs in the worker, and the store is process-local, so the endpoint would answer empty in the API process for good. Building the response therefore becomes its own minute job in DashboardFinancialService, scoped Api: it reads the newest FinancialDataLog and aggregates it, which is what the endpoint itself did before the store existed. The request path is unchanged, still a field read - no database, no parse. The expensive half stays in the worker: the aggregation producing that log entry. What this deliberately is not is a fallback for an empty store. Such a path would run once per deployment, would never be exercised, and would hide the very case it claims to cover - the reason it was rejected when the store was introduced. This job runs every minute either way. The write-through in LogJobService is removed in the same change rather than kept for a transition. Both write the same value from the same row, so a transition would only make it impossible to tell which of the two filled the store while observing the deployment. --- src/shared/services/process.service.ts | 1 + .../dashboard-financial.service.spec.ts | 62 +++++++++++++++++-- .../dashboard/dashboard-financial.service.ts | 56 ++++++++++++----- .../dashboard/latest-balance.store.ts | 14 +++-- .../log/__tests__/log-job.service.spec.ts | 25 -------- .../supporting/log/log-job.module.ts | 2 - .../supporting/log/log-job.service.ts | 17 ----- 7 files changed, 107 insertions(+), 70 deletions(-) diff --git a/src/shared/services/process.service.ts b/src/shared/services/process.service.ts index 1395ecccf2..eb13504657 100644 --- a/src/shared/services/process.service.ts +++ b/src/shared/services/process.service.ts @@ -121,6 +121,7 @@ export enum Process { DEX_PURCHASE_ORDER = 'DexPurchaseOrder', REF_CLEANUP = 'RefCleanup', JWT_REVOCATION_SYNC = 'JwtRevocationSync', + LATEST_BALANCE_CACHE = 'LatestBalanceCache', } const safetyProcesses: Process[] = [ diff --git a/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts b/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts index 02b280f7b7..39fad26132 100644 --- a/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts +++ b/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts @@ -36,6 +36,26 @@ describe('DashboardFinancialService', () => { service = module.get(DashboardFinancialService); }); + /** + * Drives the job the way it runs in production: it reads the most recent FinancialDataLog and + * resolves the assets itself, where the removed write-through was handed both in memory. + */ + async function refreshFrom( + timestamp: Date, + assetLog: AssetLog, + balancesByFinancialType: BalancesByFinancialType, + assets: Asset[], + ): Promise { + jest.spyOn(logService, 'getLatestFinancialLog').mockResolvedValue({ + id: 1, + created: timestamp, + message: JSON.stringify({ assets: assetLog, balancesByFinancialType }), + } as Log); + jest.spyOn(assetService, 'getAssetsById').mockResolvedValue(assets); + + await service.refreshLatestBalance(); + } + function mapEntry(changes: unknown) { const log = { created: new Date(), message: JSON.stringify({ changes }) } as Log; return (service as any).mapChangesLogToEntry(log); @@ -332,8 +352,8 @@ describe('DashboardFinancialService', () => { }); }); - describe('setLatestBalance (aggregation write-through)', () => { - it('aggregates byType / byBlockchain (Scrypt split, 5000 CHF Other thresholds) and writes the result into the store', () => { + describe('refreshLatestBalance (aggregation into the store)', () => { + it('aggregates byType / byBlockchain (Scrypt split, 5000 CHF Other thresholds) and writes the result into the store', async () => { // Fixture designed so every aggregation branch is exercised once: // // byType: @@ -418,12 +438,12 @@ describe('DashboardFinancialService', () => { ], }; - service.setLatestBalance(timestamp, assetLog, balancesByFinancialType, assets); + await refreshFrom(timestamp, assetLog, balancesByFinancialType, assets); expect(latestBalanceStore.set).toHaveBeenCalledWith(expected); }); - it('treats a priceless asset (priceChf: null) neutrally: neither its own blockchain group nor a shared one is skewed', () => { + it('treats a priceless asset (priceChf: null) neutrally: neither its own blockchain group nor a shared one is skewed', async () => { // asset.approxPriceChf is a nullable `double precision` column in production (144 of 430 rows // are NULL there; 136 of those have no financialType at all) -- the AssetLog[id].priceChf: number // type does not reflect that. This is a regression guard for the round-trip removed in this PR: @@ -478,9 +498,41 @@ describe('DashboardFinancialService', () => { ], }; - service.setLatestBalance(timestamp, assetLog, balancesByFinancialType, assets); + await refreshFrom(timestamp, assetLog, balancesByFinancialType, assets); expect(latestBalanceStore.set).toHaveBeenCalledWith(expected); }); + + it('leaves the store untouched when there is no log entry yet', async () => { + jest.spyOn(logService, 'getLatestFinancialLog').mockResolvedValue(undefined); + + await service.refreshLatestBalance(); + + expect(latestBalanceStore.set).not.toHaveBeenCalled(); + }); + + it('leaves the store untouched on an unparsable log entry instead of throwing', async () => { + // The job runs every minute. A single malformed entry must not take the endpoint down with + // it, nor replace a good value with a broken one. + jest + .spyOn(logService, 'getLatestFinancialLog') + .mockResolvedValue({ id: 1, created: new Date(), message: 'not json' } as Log); + + await expect(service.refreshLatestBalance()).resolves.toBeUndefined(); + + expect(latestBalanceStore.set).not.toHaveBeenCalled(); + }); + + it('does not query assets when the log entry holds none', async () => { + jest + .spyOn(logService, 'getLatestFinancialLog') + .mockResolvedValue({ id: 1, created: new Date(), message: JSON.stringify({}) } as Log); + const getAssetsByIdSpy = jest.spyOn(assetService, 'getAssetsById'); + + await service.refreshLatestBalance(); + + expect(getAssetsByIdSpy).not.toHaveBeenCalled(); + expect(latestBalanceStore.set).toHaveBeenCalled(); + }); }); }); diff --git a/src/subdomains/supporting/dashboard/dashboard-financial.service.ts b/src/subdomains/supporting/dashboard/dashboard-financial.service.ts index 8a71b34f5c..69d1a42e0c 100644 --- a/src/subdomains/supporting/dashboard/dashboard-financial.service.ts +++ b/src/subdomains/supporting/dashboard/dashboard-financial.service.ts @@ -1,8 +1,12 @@ import { Injectable } from '@nestjs/common'; +import { CronExpression } from '@nestjs/schedule'; import { Asset } from 'src/shared/models/asset/asset.entity'; import { AssetService } from 'src/shared/models/asset/asset.service'; +import { DfxLogger } from 'src/shared/services/dfx-logger'; +import { Process } from 'src/shared/services/process.service'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { RefRewardService } from '../../core/referral/reward/services/ref-reward.service'; -import { AssetLog, BalancesByFinancialType } from '../log/dto/log.dto'; +import { AssetLog, BalancesByFinancialType, FinanceLog } from '../log/dto/log.dto'; import { Log } from '../log/log.entity'; import { FinancialLogSummary } from '../log/log.repository'; import { LogService } from '../log/log.service'; @@ -19,6 +23,8 @@ import { LatestBalanceStore } from './latest-balance.store'; @Injectable() export class DashboardFinancialService { + private readonly logger = new DfxLogger(DashboardFinancialService); + constructor( private readonly logService: LogService, private readonly assetService: AssetService, @@ -126,22 +132,40 @@ export class DashboardFinancialService { } /** - * Called once a minute by LogJobService, immediately after it writes the FinancialDataLog entry - * these values are derived from. Builds the same response GET /v1/dashboard/financial/latest used - * to compute per-request (see buildLatestBalance below) and puts it in LatestBalanceStore, so the - * endpoint never touches the database again. Synchronous and DB-free by construction: assetLog, - * balancesByFinancialType and assets are exactly what the caller already holds in memory from the - * same run. A failure in here must never propagate into the caller's equity/safety-mode path — - * that isolation is the caller's responsibility (its own try/catch around this call), not this - * method's. + * Fills LatestBalanceStore from the most recent FinancialDataLog entry, so that + * GET /v1/dashboard/financial/latest keeps answering from process memory without touching the + * database - the property that turned a 23 ms median with a 1'989 ms p95 into a field read. + * + * Scope Api, because the store it fills is process-local and its only reader is that endpoint. + * The expensive part stays where it was: the financial aggregation writing the log entry runs in + * the worker, and this job only reads what that one produced. Parse and aggregation happen once + * a minute outside any request instead of on every call, which is the same work the endpoint did + * before the store existed. + * + * This is not a fallback taken only when the store is empty: such a path would run once per + * deployment and would never be exercised. It runs every minute, in normal operation as much as + * after a failure. */ - setLatestBalance( - timestamp: Date, - assetLog: AssetLog, - balancesByFinancialType: BalancesByFinancialType, - assets: Asset[], - ): void { - this.latestBalanceStore.set(this.buildLatestBalance(timestamp, assetLog, balancesByFinancialType, assets)); + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Api, process: Process.LATEST_BALANCE_CACHE }) + async refreshLatestBalance(): Promise { + const latest = await this.logService.getLatestFinancialLog(); + if (!latest) return; + + let financeLog: FinanceLog; + try { + financeLog = JSON.parse(latest.message); + } catch (e) { + this.logger.error(`Failed to parse the latest financial log (id ${latest.id}):`, e); + return; + } + + const assets = financeLog.assets + ? await this.assetService.getAssetsById(Object.keys(financeLog.assets).map(Number)) + : []; + + this.latestBalanceStore.set( + this.buildLatestBalance(latest.created, financeLog.assets, financeLog.balancesByFinancialType, assets), + ); } // Unchanged aggregation that used to run inline in getLatestBalance against a freshly parsed diff --git a/src/subdomains/supporting/dashboard/latest-balance.store.ts b/src/subdomains/supporting/dashboard/latest-balance.store.ts index 738fb3be9a..c97e6c7249 100644 --- a/src/subdomains/supporting/dashboard/latest-balance.store.ts +++ b/src/subdomains/supporting/dashboard/latest-balance.store.ts @@ -2,11 +2,15 @@ import { Injectable } from '@nestjs/common'; import { LatestBalanceResponseDto } from './dto/financial-log.dto'; /** - * Holds the single most recent LatestBalanceResponseDto, written once a minute by LogJobService - * right after it writes the FinancialDataLog entry the value is derived from, and read by - * GET /v1/dashboard/financial/latest. Exactly one entry, replaced wholesale on every job run: no - * TTL, no eviction, no size cap. There is only ever one API process instance and the writing cron - * job holds a lock, so there is never more than one writer and no cross-instance state to reconcile. + * Holds the single most recent LatestBalanceResponseDto, written once a minute by + * DashboardFinancialService.refreshLatestBalance from the newest FinancialDataLog entry, and read + * by GET /v1/dashboard/financial/latest. Exactly one entry, replaced wholesale on every job run: + * no TTL, no eviction, no size cap. + * + * The store is process-local, and so is the job filling it: it carries CronScope.Api, so it runs + * in whichever process serves the requests reading it. There is one writer per process and no + * cross-process state to reconcile - both derive the same value from the same row. + * * Empty (undefined) until the first job run after process start; the read side must not fall back * to the database in that window (see DashboardFinancialService.getLatestBalance). */ diff --git a/src/subdomains/supporting/log/__tests__/log-job.service.spec.ts b/src/subdomains/supporting/log/__tests__/log-job.service.spec.ts index daf001fa7a..d8cc449cdf 100644 --- a/src/subdomains/supporting/log/__tests__/log-job.service.spec.ts +++ b/src/subdomains/supporting/log/__tests__/log-job.service.spec.ts @@ -35,7 +35,6 @@ import { Bank } from '../../bank/bank/bank.entity'; import { BankService } from '../../bank/bank/bank.service'; import { frickCHF, frickEUR, olkyEUR, yapealCHF, yapealEUR } from '../../bank/bank/__mocks__/bank.entity.mock'; import { IbanBankName } from '../../bank/bank/dto/bank.dto'; -import { DashboardFinancialService } from '../../dashboard/dashboard-financial.service'; import { createCustomFiatOutput } from '../../fiat-output/__mocks__/fiat-output.entity.mock'; import { createCustomCryptoInput } from '../../payin/entities/__mocks__/crypto-input.entity.mock'; import { PayInService } from '../../payin/services/payin.service'; @@ -66,7 +65,6 @@ describe('LogJobService', () => { let payoutService: PayoutService; let processService: ProcessService; let paymentBalanceService: PaymentBalanceService; - let dashboardFinancialService: DashboardFinancialService; beforeEach(async () => { tradingRuleService = createMock(); @@ -89,7 +87,6 @@ describe('LogJobService', () => { payoutService = createMock(); processService = createMock(); paymentBalanceService = createMock(); - dashboardFinancialService = createMock(); const module: TestingModule = await Test.createTestingModule({ imports: [TestSharedModule], @@ -115,7 +112,6 @@ describe('LogJobService', () => { { provide: PayoutService, useValue: payoutService }, { provide: ProcessService, useValue: processService }, { provide: PaymentBalanceService, useValue: paymentBalanceService }, - { provide: DashboardFinancialService, useValue: dashboardFinancialService }, TestUtil.provideConfig(), ], }).compile(); @@ -511,29 +507,8 @@ describe('LogJobService', () => { jest .spyOn(logService, 'maxEntity') .mockResolvedValue({ message: JSON.stringify({ balancesTotal: { totalBalanceChf: 5000 } }) } as any); - // created is read as financialDataLog.created for the write-through cache call jest.spyOn(logService, 'create').mockResolvedValue({ created: new Date('2026-07-14T12:00:00Z') } as any); } - - it('keeps safety mode off, logs loudly and still resolves when setLatestBalance throws', async () => { - const errorSpy = jest.spyOn(service['logger'], 'error'); - setup(); - jest.spyOn(dashboardFinancialService, 'setLatestBalance').mockImplementation(() => { - throw new Error('cache write failed'); - }); - - await expect(service.saveTradingLog()).resolves.toBeUndefined(); - - // equity path already set safety mode correctly for the healthy book; the cache catch must not - // rethrow into the outer catch that would arm safety mode - expect(processService.setSafetyModeActive).toHaveBeenCalledWith(false); - expect(processService.setSafetyModeActive).not.toHaveBeenCalledWith(true); - - expect(errorSpy).toHaveBeenCalledWith( - 'Failed to update the latest-balance cache for the dashboard', - expect.any(Error), - ); - }); }); describe('safety mode (fail closed on non-finite total)', () => { diff --git a/src/subdomains/supporting/log/log-job.module.ts b/src/subdomains/supporting/log/log-job.module.ts index 9fce08afb3..10c25ab0f9 100644 --- a/src/subdomains/supporting/log/log-job.module.ts +++ b/src/subdomains/supporting/log/log-job.module.ts @@ -10,7 +10,6 @@ import { SellCryptoModule } from 'src/subdomains/core/sell-crypto/sell-crypto.mo import { TradingModule } from 'src/subdomains/core/trading/trading.module'; import { BankTxModule } from '../bank-tx/bank-tx.module'; import { BankModule } from '../bank/bank.module'; -import { DashboardModule } from '../dashboard/dashboard.module'; import { PayInModule } from '../payin/payin.module'; import { PayoutModule } from '../payout/payout.module'; import { LogJobService } from './log-job.service'; @@ -32,7 +31,6 @@ import { LogModule } from './log.module'; ReferralModule, PayoutModule, PaymentLinkPaymentModule, - DashboardModule, ], controllers: [], providers: [LogJobService], diff --git a/src/subdomains/supporting/log/log-job.service.ts b/src/subdomains/supporting/log/log-job.service.ts index dab36d7c99..28ce63d9df 100644 --- a/src/subdomains/supporting/log/log-job.service.ts +++ b/src/subdomains/supporting/log/log-job.service.ts @@ -40,7 +40,6 @@ import { BankTx, BankTxIndicator, BankTxType } from '../bank-tx/bank-tx/entities import { BankTxService } from '../bank-tx/bank-tx/services/bank-tx.service'; import { BankService } from '../bank/bank/bank.service'; import { IbanBankName } from '../bank/bank/dto/bank.dto'; -import { DashboardFinancialService } from '../dashboard/dashboard-financial.service'; import { CryptoInput } from '../payin/entities/crypto-input.entity'; import { PayInService } from '../payin/services/payin.service'; import { PayoutOrder, PayoutOrderContext } from '../payout/entities/payout-order.entity'; @@ -111,7 +110,6 @@ export class LogJobService { private readonly payoutService: PayoutService, private readonly processService: ProcessService, private readonly paymentBalanceService: PaymentBalanceService, - private readonly dashboardFinancialService: DashboardFinancialService, ) {} @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.TRADING_LOG, timeout: 1800 }) @@ -225,21 +223,6 @@ export class LogJobService { category: null, }); - // Write-through for GET /v1/dashboard/financial/latest: precompute here so that endpoint never - // touches the database or re-parses this message. Independent of the equity path above (which - // has already run and already armed/disarmed the safety mode correctly), so a failure here must - // never escalate to that switch: own try/catch, log loudly, never rethrow. - try { - this.dashboardFinancialService.setLatestBalance( - financialDataLog.created, - assetLog, - balancesByFinancialType, - assets, - ); - } catch (e) { - this.logger.error('Failed to update the latest-balance cache for the dashboard', e); - } - // The changeLog feeds only the informative FinancialChangesLog and is independent of the equity // path above, so it runs in its own try/catch: a reporting-price failure must not arm the equity // safety mode; the equity path above has already run and set it correctly. On failure we log the From 5ec7a887edf4e869712b27cf2128fd17c6d383c0 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:31 +0200 Subject: [PATCH 17/86] Report the worker as its own service to the collector The service name was hard-wired, so both processes would appear as one service: the trace dashboard filters on it, and while its request panels are protected by the server span kind, the panels for outgoing calls are not - they would mix the worker's blockchain, bank and exchange calls into what reads as API traffic. The name is derived from the role and reads the environment directly. Tracing starts before everything else, and importing the configuration here would pull in the instrumented modules before the SDK can patch them. The value itself is validated where the configuration is built. --- src/__tests__/tracing.spec.ts | 25 ++++++++++++++++++++++++- src/tracing.ts | 15 ++++++++++++++- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/__tests__/tracing.spec.ts b/src/__tests__/tracing.spec.ts index 908acbdfc1..54b796b101 100644 --- a/src/__tests__/tracing.spec.ts +++ b/src/__tests__/tracing.spec.ts @@ -18,7 +18,7 @@ jest.mock('@opentelemetry/sdk-metrics', () => ({ import { SpanKind, SpanStatusCode } from '@opentelemetry/api'; import { ReadableSpan } from '@opentelemetry/sdk-trace-base'; -import { ClientErrorSpanProcessor, isClientError, startTracing } from '../tracing'; +import { ClientErrorSpanProcessor, isClientError, startTracing, tracingServiceName } from '../tracing'; function fakeSpan(kind: SpanKind, statusCode: SpanStatusCode, httpStatus?: number): ReadableSpan { return { @@ -95,3 +95,26 @@ describe('startTracing', () => { expect(mockStart).toHaveBeenCalledTimes(1); }); }); + +describe('tracingServiceName', () => { + const original = process.env.CRON_ROLE; + + afterEach(() => { + if (original === undefined) delete process.env.CRON_ROLE; + else process.env.CRON_ROLE = original; + }); + + it('reports the worker under its own name', () => { + // Both processes run the same image and would otherwise report as one service, mixing the + // worker's outgoing calls into every panel not restricted to server spans. + process.env.CRON_ROLE = 'worker'; + expect(tracingServiceName()).toBe('dfx-api-worker'); + }); + + it.each(['api', 'all', undefined])('reports %p as the API service', (role) => { + if (role === undefined) delete process.env.CRON_ROLE; + else process.env.CRON_ROLE = role; + + expect(tracingServiceName()).toBe('dfx-api'); + }); +}); diff --git a/src/tracing.ts b/src/tracing.ts index 6cb61880b7..d90b653d60 100644 --- a/src/tracing.ts +++ b/src/tracing.ts @@ -95,6 +95,19 @@ export function isTelemetryEnabled(): boolean { return Boolean(process.env.OTEL_EXPORTER_OTLP_ENDPOINT); } +/** + * The name both processes report to the collector. Without the distinction they would appear as + * one service, and any panel not restricted to server spans would mix the worker's outgoing calls + * with the request traffic. + * + * Reads the environment directly rather than the configuration: tracing starts before anything + * else, and importing the configuration here would pull in the instrumented modules before the + * SDK has had a chance to patch them. The value is validated where the configuration is built. + */ +export function tracingServiceName(): string { + return process.env.CRON_ROLE === 'worker' ? 'dfx-api-worker' : 'dfx-api'; +} + export function startTracing(): NodeSDK | undefined { // Disabled unless a collector endpoint is configured (e.g. on LOC / in tests). if (!isTelemetryEnabled()) return undefined; @@ -103,7 +116,7 @@ export function startTracing(): NodeSDK | undefined { const intervalMs = metricExportIntervalMs(); sdk = new NodeSDK({ - serviceName: 'dfx-api', + serviceName: tracingServiceName(), // The 4xx-not-a-failure processor runs before the exporting batch // processor so corrected statuses are what gets exported. The exporter // reads OTEL_EXPORTER_OTLP_ENDPOINT from the environment. From bdcd817dbf0d0424cdac0edd307a09397a132216 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:32 +0200 Subject: [PATCH 18/86] Document the scope rule where it is read Every new job now carries a classification, and a wrong one fails silently on the process that reads the state - the JWT denylists were the first example, three more surfaced while working through the split. The rule therefore belongs in the contribution guide, not only in the TSDoc of the parameter. Two rules join it. The first is the one that makes a wrong scope harmless: a cache read in a request path loads on demand, and a job may refresh it but must not be the only thing filling it. AsyncCache and CachedRepository already work that way, and the jobs scoped `Both` are precisely those that do not - which is what makes them the exception rather than the pattern. The second says where periodic work may be registered at all, since a native @Cron or a bare setInterval bypasses the scope entirely. --- CONTRIBUTING.md | 61 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 66fe9f8a36..817edd55df 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -539,33 +539,58 @@ runs unconditionally and cannot be switched off without a deploy. Prefer longer intervals (15min) over aggressive polling (1min). Only use short intervals when truly needed. -#### Global vs. per-instance +#### Which process a job belongs to -The API can run as more than one instance from the same image — one serving HTTP, one running -the jobs (`CRON_JOBS_ENABLED=false` keeps an instance HTTP-only). **Every new cron job needs a -decision which of the two it is**, and getting it wrong fails silently: +The API can run as more than one process from the same image — one serving HTTP, one running the +background work — and `CRON_ROLE` decides which of them a process is (`api`, `worker`, or `all` +for a single-process setup). **Every cron job must declare which process it belongs to**, and the +compiler enforces it: `scope` is a mandatory parameter of `@DfxCron`. ```typescript -// GLOBAL (the default): writes to the database, moves money, calls an external system in a -// way that changes state. Runs on the job instance only. -@DfxCron(CronExpression.EVERY_MINUTE, { process: Process.PAYMENT }) +// Worker: writes to the database, moves money, or calls an external system in a way that +// changes state. The normal case. +@DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.PAYMENT }) async processPayments(): Promise {} -// PER-INSTANCE: effect confined to this process — refreshing an in-memory copy of global -// state, expiring a local cache, measuring this process. Runs everywhere, including the -// HTTP-only instance, because requests on that instance read what it maintains. -@DfxCron(CronExpression.EVERY_30_SECONDS, { perInstance: true }) +// Both: the effect is confined to the process it runs in — refreshing an in-memory copy of +// global state, expiring a local cache, measuring this process. It runs everywhere, because +// requests on the API process read what it maintains. +@DfxCron(CronExpression.EVERY_30_SECONDS, { scope: CronScope.Both }) async resyncDeniedJwtAccounts(): Promise {} + +// Api: maintains state read only from a request path, or drives work bound to the connections +// this process holds open. +@DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Api, process: Process.UPDATE_STATISTIC }) +async doUpdate(): Promise {} ``` -Ask: *does an HTTP handler on this instance read state that this job writes?* If yes, it is -per-instance — otherwise that state freezes at boot wherever the job does not run. The JWT -denylists are the cautionary example: frozen, they fail open and a blocked account keeps its -live tokens. +Ask: *does a request handler read state this job writes?* If both a request path and a job read +it, the answer is `Both`; if only a request path does, `Api`; otherwise `Worker`. Getting it +wrong fails silently — the state simply freezes at boot wherever the job does not run. The JWT +denylists are the cautionary example: frozen, they fail open and a blocked account keeps its live +tokens. + +A job scoped `Both` runs in every process without a shared lock, so running it twice must be +harmless by construction. If it writes to the database, sends mail, or calls a paid external API, +it is `Worker` — and if a request path needs its result, that result belongs in the database, not +in process memory. + +#### A cache read by a request path loads itself + +**A cache read in a request path must load on demand** — through `AsyncCache`, `CachedRepository` +or a lazy load of its own. A cron job may refresh it, but must not be the only thing filling it. + +This is the rule that makes a wrong scope harmless: a cache that loads itself is correct in every +process, whichever scope its refresh job carries. `AsyncCache` and `CachedRepository` already work +this way; the jobs scoped `Both` are precisely those that do not. + +#### Register periodic work through @DfxCron -Running a per-instance job twice must be harmless by construction. If it writes to the -database, sends mail, or calls a paid external API, it is global — mark it as such and, if an -HTTP path needs its result, route that result through the database rather than process memory. +`scope` only reaches jobs going through `@DfxCron`. A native `@Cron` or a bare `setInterval` is +invisible to it and therefore runs in every process — for anything writing to the database, that +means twice, without a shared lock. A test enforces this; its one exception is a timer tied to +the lifetime of a client object rather than to a schedule, and it is bound to `Config.cronRole` +directly. ### Await Discipline From f960c2c7079d4ec7447097fc17105424c3fa9162 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:33 +0200 Subject: [PATCH 19/86] Bring the cron job inventory up to date Three declarations changed and three jobs gained a flag, so the counts, the distribution and the list of jobs without a flag no longer matched the source. - 131 declarations become 134: the two transaction request jobs are now registered through @DfxCron rather than the native decorator, and the dashboard balance refresh is new. - 116 of them carry a flag rather than 110 of 131, because the DEX purchase order finalization, the referral cleanup and the JWT revocation sync each got one. - The note on multi-line declarations is corrected: mandatory scopes push many decorators past one line, so a line-based match would now miss 26, not four. The inventory deliberately does not gain a scope column. That assignment is a property of each declaration and is readable from the source at any time, whereas a copy here would be a snapshot going stale with the first job added. --- docs/cron-jobs.md | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/docs/cron-jobs.md b/docs/cron-jobs.md index ccb451d1ed..ff10be0cd9 100644 --- a/docs/cron-jobs.md +++ b/docs/cron-jobs.md @@ -1,6 +1,6 @@ # Cron jobs -Every scheduled job this service runs: **131 `@DfxCron` declarations** across 92 files and 33 areas. +Every scheduled job this service runs: **134 `@DfxCron` declarations** across 94 files and 34 areas. ## Columns @@ -13,7 +13,7 @@ Every scheduled job this service runs: **131 `@DfxCron` declarations** across 92 ## Flags -110 of the 131 jobs carry a `process` flag, 21 do not. A job with a flag can be switched off +116 of the 134 jobs carry a `process` flag, 18 do not. A job with a flag can be switched off without a deploy — `DfxCronService` skips it when the process appears in the disabled set, which `ProcessService` refreshes from the `disabledProcesses` setting and the `DISABLED_PROCESSES` environment variable every 30 seconds. @@ -21,18 +21,15 @@ environment variable every 30 seconds. A job **without** a flag runs unconditionally. That is deliberate for three of them — the `ProcessService::resync*` jobs maintain the disabled set and the JWT denylists themselves, so making them switchable would let a configuration change disable the mechanism that reads -configuration changes. For the remaining 18 it is simply an omission: +configuration changes. For the remaining 15 it is simply an omission: | Job | Interval | | --- | --- | -| `DexService::finalizePurchaseOrders` | 30 seconds | | `ExchangeController::checkTrades` | 30 seconds | | `AuthService::checkLists` | minute | -| `JwtRevocationSyncService::syncDeniedJwtAccounts` | minute | | `TransactionController::checkLists` | minute | | `UserDataService::processCleanupMailSecretCache` | minute | | `TransactionHelper::updateCache` | 5 minutes | -| `RefService::checkRefs` | hour | | `BuyService` / `SellService` / `SwapService` / `UserService` / `UserDataService` `::resetMonthlyVolumes` | 1st of month | | `BuyService` / `SellService` / `SwapService` / `UserService` / `UserDataService` `::resetAnnualVolumes` | year | @@ -45,10 +42,11 @@ New jobs should declare a flag unless there is a reason like the one above. | second | 5 | | 10 seconds | 3 | | 30 seconds | 8 | -| minute | 49 | +| minute | 51 | | 5 minutes | 17 | | 10 minutes | 15 | | hour | 16 | +| day at 3am | 1 | | day at 4am | 3 | | day at 5am | 1 | | day at 6am | 1 | @@ -62,13 +60,14 @@ Jobs by area: | Area | Jobs | Without flag | | ---- | ---: | -----------: | -| `subdomains/generic/user` | 15 | 7 | +| `subdomains/generic/user` | 15 | 6 | | `subdomains/core/monitoring` | 14 | — | | `subdomains/core/accounting` | 13 | — | | `subdomains/supporting/payin` | 12 | — | | `integration/blockchain` | 6 | — | | `subdomains/core/buy-crypto` | 6 | 4 | | `subdomains/core/sell-crypto` | 5 | 2 | +| `subdomains/supporting/payment` | 5 | 1 | | `subdomains/core/payment-link` | 4 | — | | `subdomains/generic/kyc` | 4 | — | | `subdomains/supporting/bank-tx` | 4 | — | @@ -77,9 +76,8 @@ Jobs by area: | `subdomains/supporting/support-issue` | 4 | — | | `shared` | 3 | 3 | | `subdomains/core/liquidity-management` | 3 | — | -| `subdomains/core/referral` | 3 | 1 | +| `subdomains/core/referral` | 3 | — | | `subdomains/core/trading` | 3 | — | -| `subdomains/supporting/payment` | 3 | 1 | | `subdomains/supporting/pricing` | 3 | — | | `integration/exchange` | 2 | 1 | | `subdomains/core/custody` | 2 | — | @@ -91,7 +89,8 @@ Jobs by area: | `subdomains/core/history` | 1 | 1 | | `subdomains/core/statistic` | 1 | — | | `subdomains/generic/admin` | 1 | — | -| `subdomains/supporting/dex` | 1 | 1 | +| `subdomains/supporting/dashboard` | 1 | — | +| `subdomains/supporting/dex` | 1 | — | | `subdomains/supporting/fiat-payin` | 1 | — | | `subdomains/supporting/notification` | 1 | — | | `subdomains/supporting/payout` | 1 | — | @@ -99,8 +98,8 @@ Jobs by area: ## How this list is produced Every `@DfxCron(` occurrence in `src/**/*.ts`. Decorator arguments are read by a balanced-paren -scan, so multi-line declarations are included — a line-based match misses four of them. The parsed -count is asserted against a raw text count of the decorator: **131 = 131**, no gap. Class and +scan, so multi-line declarations are included — a line-based match misses 26 of them. The parsed +count is asserted against a raw text count of the decorator: **134 = 134**, no gap. Class and method come from the enclosing `export class` (including `export abstract class`) and the identifier following the decorator. @@ -126,7 +125,7 @@ the interval while running as an independent timer with its own lock. | 10 seconds | `MONITOR_EVENT_LOOP` | `MonitorEventLoopService::monitorEventLoop` | `subdomains/core/monitoring/monitor-event-loop.service.ts` | | 30 seconds | `LNURL_AUTH_CACHE` | `AuthLnUrlService::processCleanupAccessToken` | `subdomains/generic/user/models/auth/auth-lnurl.service.ts` | | 30 seconds | `BANK_TX` | `BankTxService::checkBankTx` | `subdomains/supporting/bank-tx/bank-tx/services/bank-tx.service.ts` | -| 30 seconds | — | `DexService::finalizePurchaseOrders` | `subdomains/supporting/dex/services/dex.service.ts` | +| 30 seconds | `DEX_PURCHASE_ORDER` | `DexService::finalizePurchaseOrders` | `subdomains/supporting/dex/services/dex.service.ts` | | 30 seconds | — | `ExchangeController::checkTrades` | `integration/exchange/controllers/exchange.controller.ts` | | 30 seconds | `PAY_OUT` | `PayoutService::processOrders` | `subdomains/supporting/payout/services/payout.service.ts` | | 30 seconds | — | `ProcessService::resyncDeniedJwtAccounts` | `shared/services/process.service.ts` | @@ -146,10 +145,11 @@ the interval while running as an independent timer with its own lock. | minute | `MONITORING` | `CheckoutObserver::fetch` | `subdomains/core/monitoring/observers/checkout.observer.ts` | | minute | `PAY_IN` | `CitreaBaseStrategy::checkPayInEntries` | `subdomains/supporting/payin/strategies/register/impl/base/citrea.strategy.ts` | | minute | `CUSTODY` | `CustodyJobService::handleOrders` | `subdomains/core/custody/services/custody-job.service.ts` | +| minute | `LATEST_BALANCE_CACHE` | `DashboardFinancialService::refreshLatestBalance` | `subdomains/supporting/dashboard/dashboard-financial.service.ts` | | minute | `FIAT_OUTPUT` | `FiatOutputJobService::fillFiatOutput` | `subdomains/supporting/fiat-output/fiat-output-job.service.ts` | | minute | `FIAT_PAY_IN` | `FiatPayInSyncService::syncCheckout` | `subdomains/supporting/fiat-payin/services/fiat-payin-sync.service.ts` | | minute | `PAY_IN` | `InternetComputerStrategy::checkPayInEntries` | `subdomains/supporting/payin/strategies/register/impl/icp.strategy.ts` | -| minute | — | `JwtRevocationSyncService::syncDeniedJwtAccounts` | `subdomains/generic/user/models/user-data/jwt-revocation-sync.service.ts` | +| minute | `JWT_REVOCATION_SYNC` | `JwtRevocationSyncService::syncDeniedJwtAccounts` | `subdomains/generic/user/models/user-data/jwt-revocation-sync.service.ts` | | minute | `KYC` | `KycService::reviewKycSteps` | `subdomains/generic/kyc/services/kyc.service.ts` | | minute | `LEDGER_BOOKING_BANK_TX` | `LedgerBookingJobService::runBankTx` | `subdomains/core/accounting/services/ledger-booking-job.service.ts` | | minute | `LEDGER_BOOKING_BUY_CRYPTO` | `LedgerBookingJobService::runBuyCrypto` | `subdomains/core/accounting/services/ledger-booking-job.service.ts` | @@ -178,6 +178,7 @@ the interval while running as an independent timer with its own lock. | minute | `TRADING` | `TradingJobService::processRules` | `subdomains/core/trading/services/trading-job.service.ts` | | minute | — | `TransactionController::checkLists` | `subdomains/core/history/controllers/transaction.controller.ts` | | minute | `TX_MAIL` | `TransactionNotificationService::sendNotificationMails` | `subdomains/supporting/payment/services/transaction-notification.service.ts` | +| minute | `TX_REQUEST` | `TransactionRequestService::txRequestStatusSync` | `subdomains/supporting/payment/services/transaction-request.service.ts` | | minute | `USER_DATA` | `UserDataJobService::fillUserData` | `subdomains/generic/user/models/user-data/user-data-job.service.ts` | | minute | — | `UserDataService::processCleanupMailSecretCache` | `subdomains/generic/user/models/user-data/user-data.service.ts` | | minute | `USER` | `UserJobService::fillUser` | `subdomains/generic/user/models/user/user-job.service.ts` | @@ -224,11 +225,12 @@ the interval while running as an independent timer with its own lock. | hour | `PRICING` | `FiatPricesService::updatePrices` | `subdomains/supporting/pricing/services/fiat-prices.service.ts` | | hour | `KYC_MAIL` | `KycNotificationService::sendNotificationMails` | `subdomains/generic/kyc/services/kyc-notification.service.ts` | | hour | `PAYMENT_FORWARDING` | `PaymentCronService::forwardDeposits` | `subdomains/core/payment-link/services/payment-cron.service.ts` | -| hour | — | `RefService::checkRefs` | `subdomains/core/referral/process/ref.service.ts` | +| hour | `REF_CLEANUP` | `RefService::checkRefs` | `subdomains/core/referral/process/ref.service.ts` | | hour | `UPDATE_STATISTIC` | `StatisticService::doUpdate` | `subdomains/core/statistic/statistic.service.ts` | | hour | `SUPPORT_BOT` | `SupportIssueJobService::autoOnHold` | `subdomains/supporting/support-issue/services/support-issue-job.service.ts` | | hour | `BLACK_SQUAD_MAIL` | `UserDataNotificationService::sendNotificationMails` | `subdomains/generic/user/models/user-data/user-data-notification.service.ts` | | hour | `VIRTUAL_IBAN_FRICK_ISSUANCE_RECONCILIATION` | `VirtualIbanFrickIssuanceReconciliationService::reconcileRetiredIssuanceReferences` | `subdomains/supporting/bank/virtual-iban/virtual-iban-frick-issuance-reconciliation.service.ts` | +| day at 3am | `TX_REQUEST_WAITING_EXPIRY` | `TransactionRequestService::txRequestWaitingExpiryCheck` | `subdomains/supporting/payment/services/transaction-request.service.ts` | | day at 4am | `CUSTODY` | `CustodyJobService::resetExpiredConfirmedOrders` | `subdomains/core/custody/services/custody-job.service.ts` | | day at 4am | `KYC` | `KycService::checkIdentSteps` | `subdomains/generic/kyc/services/kyc.service.ts` | | day at 4am | `LEDGER_MARK_TO_MARKET` | `LedgerMarkToMarketService::run` | `subdomains/core/accounting/services/ledger-mark-to-market.service.ts` | From abc7b7f3ca10788d2f27444d0473e72b527a8a56 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:35 +0200 Subject: [PATCH 20/86] Cover the role binding of the Spark maintenance timer The client's test replaces the whole configuration module so it does not pull in the configuration chain, and the replacement exported GetConfig only. Reading the role from it therefore threw at construction time - every test building a client failed, including the ones about the broadcast boundary. The mock now carries CronRole as well, and the role it reports is a variable the tests can set. Two cases assert what the binding is for: the timer starts in the worker with its five-minute interval, and it does not start in the API process. Without that check both processes would drive on-chain wallet maintenance against the same seed. --- .../spark/__tests__/spark-client.spec.ts | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/integration/blockchain/spark/__tests__/spark-client.spec.ts b/src/integration/blockchain/spark/__tests__/spark-client.spec.ts index 798c858c07..5cef81dfef 100644 --- a/src/integration/blockchain/spark/__tests__/spark-client.spec.ts +++ b/src/integration/blockchain/spark/__tests__/spark-client.spec.ts @@ -15,8 +15,17 @@ jest.mock('@buildonspark/spark-sdk', () => ({ SparkWallet: { initialize: jest.fn() }, })); +// Rolle des Prozesses, veränderbar je Test. Der Name muss mit `mock` beginnen, +// sonst verbietet Jest den Zugriff aus der gehobenen Modul-Fabrik heraus. +let mockCronRole = 'worker'; + jest.mock('src/config/config', () => ({ + // Das Modul wird vollständig ersetzt, damit der Test nicht die ganze + // Konfigurationskette lädt. CronRole muss deshalb hier mitkommen: der Client + // liest es, um den Optimierungs-Timer an die Rolle zu binden. + CronRole: { All: 'all', Api: 'api', Worker: 'worker' }, GetConfig: () => ({ + cronRole: mockCronRole, blockchain: { spark: { sparkWalletSeed: 'test seed phrase', @@ -63,6 +72,36 @@ describe('SparkClient', () => { afterEach(() => { jest.restoreAllMocks(); + mockCronRole = 'worker'; + }); + + // --- TOKEN OPTIMIZATION TIMER --- // + + describe('token optimization timer', () => { + it('runs the wallet maintenance in the worker process', () => { + // On-chain wallet maintenance is global work and must run in exactly one + // process. The timer predates the scheduler and bypasses it, so it carries + // the role check itself. + const interval = jest.spyOn(global, 'setInterval'); + + new SparkClient(); + + expect(interval).toHaveBeenCalledTimes(1); + expect(interval.mock.calls[0][1]).toBe(5 * 60 * 1000); + + clearInterval(interval.mock.results[0].value as NodeJS.Timeout); + }); + + it('does not run it in the API process', () => { + // Both processes run the same image against the same seed: without this + // check the maintenance would run twice, unsynchronised. + mockCronRole = 'api'; + const interval = jest.spyOn(global, 'setInterval'); + + new SparkClient(); + + expect(interval).not.toHaveBeenCalled(); + }); }); describe('sendTransaction', () => { From 9bd75de5e666ea2baa71b446467ae3e29874fa60 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:36 +0200 Subject: [PATCH 21/86] Drop what the removed write-through left behind The log job no longer reads the created entry, and the test helper belonged to the case where the write-through could throw. Both linted as unused. --- .../log/__tests__/log-job.service.spec.ts | 19 ------------------- .../supporting/log/log-job.service.ts | 2 +- 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/src/subdomains/supporting/log/__tests__/log-job.service.spec.ts b/src/subdomains/supporting/log/__tests__/log-job.service.spec.ts index d8cc449cdf..ec8304ea4a 100644 --- a/src/subdomains/supporting/log/__tests__/log-job.service.spec.ts +++ b/src/subdomains/supporting/log/__tests__/log-job.service.spec.ts @@ -492,25 +492,6 @@ describe('LogJobService', () => { }); }); - describe('latest-balance cache write isolation (cache failure must not arm the equity safety mode)', () => { - // a healthy, finite book comfortably above the minimum -> the equity path leaves safety mode off - function setup() { - jest.spyOn(service as any, 'getTradingLog').mockResolvedValue({}); - jest.spyOn(service as any, 'getAssetLog').mockResolvedValue({}); - jest - .spyOn(service as any, 'getBalancesByFinancialType') - .mockReturnValue({ EUR: { plusBalance: 5000, plusBalanceChf: 5000, minusBalance: 0, minusBalanceChf: 0 } }); - jest.spyOn(service as any, 'getChangeLog').mockResolvedValue({}); - jest.spyOn(assetService, 'getAssetsWith').mockResolvedValue([] as any); - jest.spyOn(settingService, 'getObj').mockResolvedValue(100 as any); - jest.spyOn(refRewardService, 'getOpenRefCreditLiability').mockResolvedValue({ amountEur: 0, amountChf: 0 }); - jest - .spyOn(logService, 'maxEntity') - .mockResolvedValue({ message: JSON.stringify({ balancesTotal: { totalBalanceChf: 5000 } }) } as any); - jest.spyOn(logService, 'create').mockResolvedValue({ created: new Date('2026-07-14T12:00:00Z') } as any); - } - }); - describe('safety mode (fail closed on non-finite total)', () => { function setup(buckets: Record, minTotalBalanceChf: number) { jest.spyOn(service as any, 'getTradingLog').mockResolvedValue({}); diff --git a/src/subdomains/supporting/log/log-job.service.ts b/src/subdomains/supporting/log/log-job.service.ts index 28ce63d9df..d652163225 100644 --- a/src/subdomains/supporting/log/log-job.service.ts +++ b/src/subdomains/supporting/log/log-job.service.ts @@ -189,7 +189,7 @@ export class LogJobService { const btcAssetPriceChf = btcAsset ? assetLog[btcAsset.id]?.priceChf : undefined; const btcPriceChfColumn = btcAssetPriceChf != null && Number.isFinite(btcAssetPriceChf) ? btcAssetPriceChf : null; - const financialDataLog = await this.logService.create({ + await this.logService.create({ system: 'LogService', subsystem: 'FinancialDataLog', severity: LogSeverity.INFO, From d586cfaf7966fcf6b511f4809ef7acc0489cb66d Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:38 +0200 Subject: [PATCH 22/86] Serialise the monitoring write instead of only merging it Read-merge-write without a lock narrows the race instead of closing it. Two writers that both read before either wrote still overwrite each other, and the lost value does not come back on its own: the next run compares against this process own previous state, finds the metric unchanged, and writes nothing. A value that rarely changes - a health flag, for instance - would stay wrong in the row until it changes for real. The window is not theoretical either, since both processes debounce their updates by the same two seconds and many observers are bound to the same cron boundaries. Read, merge and write now happen in one transaction that takes a write lock on the row, so a second writer waits instead of reading a value it is about to destroy. The test asserts the property that matters: the row is read under a write lock, inside the same transaction that writes it. The repository is mocked through the repo type rather than a hand-rolled object, which also removes the `any` the project rules forbid. --- .../__tests__/monitoring.service.spec.ts | 59 +++++++++++++++---- .../core/monitoring/monitoring.service.ts | 36 ++++++++--- 2 files changed, 78 insertions(+), 17 deletions(-) diff --git a/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts b/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts index af6ce546e0..98d7989045 100644 --- a/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts +++ b/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts @@ -1,8 +1,9 @@ -import { createMock } from '@golevelup/ts-jest'; +import { DeepMocked, createMock } from '@golevelup/ts-jest'; import { NotFoundException } from '@nestjs/common'; import { NotificationService } from 'src/subdomains/supporting/notification/services/notification.service'; import { MonitoringService } from '../monitoring.service'; import { Metric, SystemState } from '../system-state-snapshot.entity'; +import { SystemStateSnapshotRepository } from '../system-state-snapshot.repository'; function snapshot(state: SystemState): { id: number; data: string } { return { id: 1, data: JSON.stringify(state) }; @@ -13,7 +14,7 @@ function metric(data: unknown, updated: string): Metric { } describe('MonitoringService', () => { - let repo: any; + let repo: DeepMocked; let notificationService: NotificationService; let service: MonitoringService; @@ -27,8 +28,32 @@ describe('MonitoringService', () => { // identically in the response. const asRead = (): SystemState => JSON.parse(JSON.stringify(persisted)); + let lockedReads: unknown[]; + let written: { id: number; data: string }[]; + beforeEach(() => { - repo = { findOne: jest.fn().mockResolvedValue(snapshot(persisted)), save: jest.fn().mockResolvedValue(undefined) }; + lockedReads = []; + written = []; + + repo = createMock(); + repo.findOne.mockResolvedValue(snapshot(persisted) as never); + + // Stands in for the transaction: same row content, but through a manager whose lock option + // and writes the test can inspect. + const manager = { + findOne: jest.fn().mockImplementation((_entity: unknown, options: { lock?: unknown }) => { + lockedReads.push(options?.lock); + return Promise.resolve(written.length ? written[written.length - 1] : snapshot(persisted)); + }), + save: jest.fn().mockImplementation((_entity: unknown, row: { id: number; data: string }) => { + written.push(row); + return Promise.resolve(row); + }), + }; + Object.defineProperty(repo, 'manager', { + value: { transaction: (run: (m: unknown) => Promise) => run(manager) }, + configurable: true, + }); notificationService = createMock(); service = new MonitoringService(repo, notificationService); @@ -100,16 +125,30 @@ describe('MonitoringService', () => { await service['persist'](prev, next); - const written = JSON.parse(repo.save.mock.calls[0][0].data) as SystemState; + const result = JSON.parse(written[0].data) as SystemState; - expect(written.bank.balance.data).toEqual({ chf: 43 }); - expect(written.node.health.data).toEqual({ up: true }); + expect(result.bank.balance.data).toEqual({ chf: 43 }); + expect(result.node.health.data).toEqual({ up: true }); }); it('writes nothing when no metric changed', async () => { await service['persist'](persisted, persisted); - expect(repo.save).not.toHaveBeenCalled(); + expect(written).toEqual([]); + }); + + it('reads the row under a write lock, in the same transaction it writes in', async () => { + // Without the lock, merging only narrows the race instead of closing it: two writers that + // both read before either wrote still overwrite each other - and the lost value does not + // come back on its own, because the next run compares it against this process's own + // previous state and finds it unchanged. + const prev: SystemState = { bank: { balance: metric({ chf: 42 }, '2020-01-01T00:10:00Z') } }; + const next: SystemState = { bank: { balance: metric({ chf: 43 }, '2020-01-01T00:20:00Z') } }; + + await service['persist'](prev, next); + + expect(lockedReads).toEqual([{ mode: 'pessimistic_write' }]); + expect(written).toHaveLength(1); }); it('writes a metric that did not exist before', async () => { @@ -117,10 +156,10 @@ describe('MonitoringService', () => { await service['persist']({}, next); - const written = JSON.parse(repo.save.mock.calls[0][0].data) as SystemState; + const result = JSON.parse(written[0].data) as SystemState; - expect(written.ledger.open.data).toEqual({ count: 3 }); - expect(written.bank.balance.data).toEqual({ chf: 42 }); + expect(result.ledger.open.data).toEqual({ count: 3 }); + expect(result.bank.balance.data).toEqual({ chf: 42 }); }); }); }); diff --git a/src/subdomains/core/monitoring/monitoring.service.ts b/src/subdomains/core/monitoring/monitoring.service.ts index 97831ee554..31e9b890e6 100644 --- a/src/subdomains/core/monitoring/monitoring.service.ts +++ b/src/subdomains/core/monitoring/monitoring.service.ts @@ -5,7 +5,14 @@ import { DfxLogger } from 'src/shared/services/dfx-logger'; import { MailContext, MailType } from 'src/subdomains/supporting/notification/enums'; import { NotificationService } from 'src/subdomains/supporting/notification/services/notification.service'; import { MetricObserver } from './metric.observer'; -import { Metric, MetricName, SubsystemName, SubsystemState, SystemState } from './system-state-snapshot.entity'; +import { + Metric, + MetricName, + SubsystemName, + SubsystemState, + SystemState, + SystemStateSnapshot, +} from './system-state-snapshot.entity'; import { SystemStateSnapshotRepository } from './system-state-snapshot.repository'; type SubsystemObservers = Map>; @@ -131,20 +138,35 @@ export class MonitoringService implements OnModuleInit { * process wrote in the meantime - the API process would overwrite the observers' work with its * boot state, and the webhook path the other way round. Merging the changed metrics into the * stored row makes it irrelevant which process writes. + * + * Read, merge and write happen inside one transaction that locks the row. Without the lock the + * merge only narrows the race instead of closing it: two writers that read before either wrote + * still overwrite each other, and the lost value does not come back on its own - the next run + * compares against this process's own previous state, finds the metric unchanged, and writes + * nothing. A value that rarely changes would stay wrong in the row until it does. */ private async persist(prevState: SystemState, newState: SystemState) { try { const changed = this.changedMetrics(prevState, newState); if (!changed.length) return; - const stored = (await this.readState()) ?? {}; - const merged = cloneDeep(stored); + const merged = await this.systemStateSnapshotRepo.manager.transaction(async (manager) => { + const row = await manager.findOne(SystemStateSnapshot, { + where: { id: 1 }, + lock: { mode: 'pessimistic_write' }, + }); - for (const [subsystem, metric] of changed) { - merged[subsystem] = { ...(merged[subsystem] ?? {}), [metric]: newState[subsystem][metric] }; - } + const stored: SystemState = row ? JSON.parse(row.data) : {}; + const state = cloneDeep(stored); + + for (const [subsystem, metric] of changed) { + state[subsystem] = { ...(state[subsystem] ?? {}), [metric]: newState[subsystem][metric] }; + } + + await manager.save(SystemStateSnapshot, { id: 1, data: JSON.stringify(state) }); - await this.systemStateSnapshotRepo.save({ id: 1, data: JSON.stringify(merged) }); + return state; + }); this.#storedState = { state: merged, loaded: Date.now() }; } catch (e) { From bed4fa729acbcf4c2f3b28a6d7a218768c5bfb5e Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:40 +0200 Subject: [PATCH 23/86] Follow the repository conventions the review measured Four points from the review round, each verified against the repository rather than taken on trust: - Enum members follow the repository convention: 335 enums use ALL_CAPS members against 6 using PascalCase, two of which were the ones added here. CronRole and CronScope now match the majority. The Spark test carries a hand-written stand-in for CronRole, which had to follow - and did not, which the test caught. - docs/cron-jobs.md gains a Scope column and a section on the distribution. The earlier reasoning, that a column would go stale, applies to interval and flag just as much, and those are maintained. All three are read from the decorator arguments, so the document is as accurate as the source. - The two caches that only a request path fills and reads - the exchange trade map and the refund list, both controller fields - are api-scoped, not both. Under `both` the job would run in a process where the map is always empty. - Comments in the Spark test are English, like the rest of this repository. Also from checking my own comments against the rule that a comment may only claim what a reader can verify in this repository: production measurements, a claim about how the code came to be, references to a dashboard living elsewhere, and an assertion about the deployment topology are gone. Each was replaced by the property the code itself carries, not by a weaker version of the same claim. The balance refresh job now states its cost honestly: one extra read per minute compared to the write-through it replaces, in the single-process role as well. The Spark timer gains the case the whole change rests on - the single-process role, where the timer must still run. --- CONTRIBUTING.md | 2 +- docs/cron-jobs.md | 292 ++++++++++-------- src/__tests__/tracing.spec.ts | 4 +- src/config/__tests__/cron-role.config.spec.ts | 12 +- src/config/config.ts | 6 +- .../services/binance-pay.service.ts | 2 +- .../blockchain/deuro/deuro.service.ts | 2 +- .../frankencoin/frankencoin.service.ts | 2 +- .../blockchain/juice/juice.service.ts | 2 +- .../shared/evm/evm-decimals.service.ts | 2 +- .../blockchain-config-check.service.ts | 2 +- .../spark/__tests__/spark-client.spec.ts | 25 +- .../blockchain/spark/spark-client.ts | 2 +- .../blockchain/zano/services/zano.service.ts | 2 +- .../controllers/exchange.controller.ts | 5 +- .../exchange/services/exchange-tx.service.ts | 2 +- .../__tests__/cron-registration.guard.spec.ts | 3 +- .../__tests__/dfx-cron.service.spec.ts | 8 +- src/shared/services/dfx-cron.service.ts | 10 +- src/shared/services/process.service.ts | 6 +- src/shared/utils/cron.ts | 6 +- .../services/ledger-booking-job.service.ts | 20 +- .../services/ledger-cutover.service.ts | 2 +- .../services/ledger-mark-to-market.service.ts | 2 +- .../services/ledger-reconciliation.service.ts | 2 +- .../core/aml/services/sanction.service.ts | 2 +- .../services/buy-crypto-job.service.ts | 4 +- .../core/buy-crypto/routes/buy/buy.service.ts | 4 +- .../buy-crypto/routes/swap/swap.service.ts | 4 +- .../custody/services/custody-job.service.ts | 4 +- .../services/faucet-request.service.ts | 2 +- .../controllers/transaction.controller.ts | 4 +- .../liquidity-management-pipeline.service.ts | 2 +- .../liquidity-management-rule.service.ts | 2 +- .../services/liquidity-management.service.ts | 2 +- .../monitor-connection-pool.service.ts | 4 +- .../monitoring/monitor-event-loop.service.ts | 2 +- .../core/monitoring/observers/aml.observer.ts | 2 +- .../monitoring/observers/bank.observer.ts | 2 +- .../monitoring/observers/checkout.observer.ts | 2 +- .../monitoring/observers/exchange.observer.ts | 2 +- .../observers/external-services.observer.ts | 2 +- .../observers/liquidity.observer.ts | 2 +- .../observers/node-balance.observer.ts | 2 +- .../observers/node-health.observer.ts | 2 +- .../monitoring/observers/payment.observer.ts | 2 +- .../observers/realunit-w2w-gas.observer.ts | 2 +- .../monitoring/observers/user.observer.ts | 2 +- .../services/payment-cron.service.ts | 14 +- .../services/payment-link-fee.service.ts | 2 +- .../core/referral/process/ref.service.ts | 2 +- .../reward/services/ref-reward-job.service.ts | 4 +- .../process/services/buy-fiat-job.service.ts | 4 +- .../services/buy-fiat-notification.service.ts | 2 +- .../core/sell-crypto/route/sell.service.ts | 4 +- .../core/statistic/statistic.service.ts | 2 +- .../trading/services/trading-job.service.ts | 6 +- src/subdomains/generic/admin/admin.service.ts | 2 +- .../kyc/services/kyc-notification.service.ts | 2 +- .../generic/kyc/services/kyc.service.ts | 4 +- .../generic/kyc/services/tfa.service.ts | 2 +- .../user/models/auth/auth-lnurl.service.ts | 4 +- .../generic/user/models/auth/auth.service.ts | 2 +- .../models/bank-data/bank-data.service.ts | 2 +- .../organization/organization.service.ts | 2 +- .../user-data/jwt-revocation-sync.service.ts | 2 +- .../models/user-data/user-data-job.service.ts | 2 +- .../user-data-notification.service.ts | 2 +- .../models/user-data/user-data.service.ts | 6 +- .../user/models/user/user-job.service.ts | 2 +- .../generic/user/models/user/user.service.ts | 4 +- .../webhook/webhook-notification.service.ts | 2 +- .../bank-tx-return-notification.service.ts | 2 +- .../bank-tx-return/bank-tx-return.service.ts | 2 +- .../bank-tx/services/bank-tx.service.ts | 4 +- .../bank/bank-account/bank-account.service.ts | 6 +- ...n-frick-issuance-reconciliation.service.ts | 2 +- .../dashboard/dashboard-financial.service.ts | 9 +- .../dashboard/latest-balance.store.ts | 2 +- .../supporting/dex/services/dex.service.ts | 2 +- .../fiat-output/fiat-output-frick.service.ts | 2 +- .../fiat-output/fiat-output-job.service.ts | 6 +- .../services/fiat-payin-sync.service.ts | 2 +- .../supporting/log/log-job.service.ts | 2 +- src/subdomains/supporting/log/log.service.ts | 2 +- .../services/notification-job.service.ts | 2 +- .../services/payin-notification.service.ts | 2 +- .../payin/services/payin.service.ts | 8 +- .../register/impl/base/citrea.strategy.ts | 2 +- .../register/impl/bitcoin.strategy.ts | 2 +- .../register/impl/cardano.strategy.ts | 2 +- .../strategies/register/impl/firo.strategy.ts | 2 +- .../strategies/register/impl/icp.strategy.ts | 2 +- .../register/impl/monero.strategy.ts | 2 +- .../strategies/register/impl/zano.strategy.ts | 2 +- .../payment/services/fee.service.ts | 2 +- .../payment/services/transaction-helper.ts | 2 +- .../transaction-notification.service.ts | 2 +- .../services/transaction-request.service.ts | 4 +- .../payout/services/payout.service.ts | 2 +- .../services/asset-prices-job.service.ts | 4 +- .../pricing/services/fiat-prices.service.ts | 2 +- .../realunit/realunit-job.service.ts | 4 +- .../limit-request-notification.service.ts | 2 +- .../services/support-escalation.service.ts | 2 +- .../services/support-issue-job.service.ts | 4 +- src/tracing.ts | 6 +- 107 files changed, 353 insertions(+), 313 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 817edd55df..89c84d6eed 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -534,7 +534,7 @@ async processPayments(): Promise { Declare a `process` flag unless the job maintains the disabled set itself. Without one the job runs unconditionally and cannot be switched off without a deploy. -[docs/cron-jobs.md](docs/cron-jobs.md) lists every scheduled job with its interval and flag. +[docs/cron-jobs.md](docs/cron-jobs.md) lists every scheduled job with its interval, flag and scope. **Adding, removing or re-scheduling a job must be reflected there in the same PR.** Prefer longer intervals (15min) over aggressive polling (1min). Only use short intervals when truly needed. diff --git a/docs/cron-jobs.md b/docs/cron-jobs.md index ff10be0cd9..79ca8b144d 100644 --- a/docs/cron-jobs.md +++ b/docs/cron-jobs.md @@ -8,9 +8,26 @@ Every scheduled job this service runs: **134 `@DfxCron` declarations** across 94 | ------ | ------- | | **Interval** | The `CronExpression` / `CustomCronExpression` the job is registered with | | **Flag** | The `process:` kill switch that disables the job at runtime. `—` means the job has none and always runs | +| **Scope** | Which process registers the job: `worker`, `api`, or `both` | | **Job** | Class and method | | **File** | Path below `src/` | +## Scopes + +`scope` is a mandatory parameter of `@DfxCron` and says which process registers the job: +115 are `worker`, 6 are `api`, 13 are `both`. `CRON_ROLE` decides what a process is +(`worker`, `api`, or `all` for a single-process setup); a process runs its own scope plus `both`. + +`worker` is the normal case — anything writing to the database or driving business forward belongs +to exactly one process. `both` is for a job maintaining process-local state that a request path +also reads, so it must run everywhere; running it twice has to be harmless by construction, which +rules out database writes, mail and paid external calls. `api` is for state read only from a +request path, or for work bound to the connections that process holds open. + +Getting the scope wrong fails silently: the cache a job maintains simply stays empty in the +process that reads it. The rule that keeps that harmless is in CONTRIBUTING.md — a cache read in a +request path loads on demand, and a job may refresh it but must not be the only thing filling it. + ## Flags 116 of the 134 jobs carry a `process` flag, 18 do not. A job with a flag can be switched off @@ -98,7 +115,8 @@ Jobs by area: ## How this list is produced Every `@DfxCron(` occurrence in `src/**/*.ts`. Decorator arguments are read by a balanced-paren -scan, so multi-line declarations are included — a line-based match misses 26 of them. The parsed +scan, so multi-line declarations are included — a line-based match misses 26 of them. Interval, +flag and scope come from those arguments, so all three are as accurate as the source. The parsed count is asserted against a raw text count of the decorator: **134 = 134**, no gap. Class and method come from the enclosing `export class` (including `export abstract class`) and the identifier following the decorator. @@ -113,139 +131,139 @@ the interval while running as an independent timer with its own lock. ## Jobs -| Interval | Flag | Job | File | -| -------- | ---- | --- | ---- | -| second | `PAY_IN` | `BitcoinStrategy::checkPayInEntries` | `subdomains/supporting/payin/strategies/register/impl/bitcoin.strategy.ts` | -| second | `PAY_IN` | `FiroStrategy::checkPayInEntries` | `subdomains/supporting/payin/strategies/register/impl/firo.strategy.ts` | -| second | `PAY_IN` | `MoneroStrategy::checkPayInEntries` | `subdomains/supporting/payin/strategies/register/impl/monero.strategy.ts` | -| second | `MONITOR_CONNECTION_POOL` | `MonitorConnectionPoolService::monitorConnectionPool` | `subdomains/core/monitoring/monitor-connection-pool.service.ts` | -| second | `PAY_IN` | `ZanoStrategy::checkPayInEntries` | `subdomains/supporting/payin/strategies/register/impl/zano.strategy.ts` | -| 10 seconds | `LIQUIDITY_MANAGEMENT` | `LiquidityManagementPipelineService::processPipelines` | `subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.ts` | -| 10 seconds | `MONITOR_CONNECTION_POOL` | `MonitorConnectionPoolService::monitorConnectionPoolStatic` | `subdomains/core/monitoring/monitor-connection-pool.service.ts` | -| 10 seconds | `MONITOR_EVENT_LOOP` | `MonitorEventLoopService::monitorEventLoop` | `subdomains/core/monitoring/monitor-event-loop.service.ts` | -| 30 seconds | `LNURL_AUTH_CACHE` | `AuthLnUrlService::processCleanupAccessToken` | `subdomains/generic/user/models/auth/auth-lnurl.service.ts` | -| 30 seconds | `BANK_TX` | `BankTxService::checkBankTx` | `subdomains/supporting/bank-tx/bank-tx/services/bank-tx.service.ts` | -| 30 seconds | `DEX_PURCHASE_ORDER` | `DexService::finalizePurchaseOrders` | `subdomains/supporting/dex/services/dex.service.ts` | -| 30 seconds | — | `ExchangeController::checkTrades` | `integration/exchange/controllers/exchange.controller.ts` | -| 30 seconds | `PAY_OUT` | `PayoutService::processOrders` | `subdomains/supporting/payout/services/payout.service.ts` | -| 30 seconds | — | `ProcessService::resyncDeniedJwtAccounts` | `shared/services/process.service.ts` | -| 30 seconds | — | `ProcessService::resyncDeniedJwtAddresses` | `shared/services/process.service.ts` | -| 30 seconds | — | `ProcessService::resyncDisabledProcesses` | `shared/services/process.service.ts` | -| minute | `PAY_OUT` | `AdminService::completeLiquidityOrders` | `subdomains/generic/admin/admin.service.ts` | -| minute | `MONITORING` | `AmlObserver::fetch` | `subdomains/core/monitoring/observers/aml.observer.ts` | -| minute | — | `AuthService::checkLists` | `subdomains/generic/user/models/auth/auth.service.ts` | -| minute | `BANK_DATA_VERIFICATION` | `BankDataService::checkAndSetActive` | `subdomains/generic/user/models/bank-data/bank-data.service.ts` | -| minute | `MONITORING` | `BankObserver::fetch` | `subdomains/core/monitoring/observers/bank.observer.ts` | -| minute | `BANK_TX_RETURN_MAIL` | `BankTxReturnNotificationService::sendBankTxReturnMail` | `subdomains/supporting/bank-tx/bank-tx-return/bank-tx-return-notification.service.ts` | -| minute | `BUY_CRYPTO` | `BuyCryptoJobService::process` | `subdomains/core/buy-crypto/process/services/buy-crypto-job.service.ts` | -| minute | `BUY_FIAT` | `BuyFiatJobService::addFiatOutputs` | `subdomains/core/sell-crypto/process/services/buy-fiat-job.service.ts` | -| minute | `BUY_FIAT` | `BuyFiatJobService::checkCryptoPayIn` | `subdomains/core/sell-crypto/process/services/buy-fiat-job.service.ts` | -| minute | `BUY_FIAT_MAIL` | `BuyFiatNotificationService::sendNotificationMails` | `subdomains/core/sell-crypto/process/services/buy-fiat-notification.service.ts` | -| minute | `PAY_IN` | `CardanoStrategy::checkPayInEntries` | `subdomains/supporting/payin/strategies/register/impl/cardano.strategy.ts` | -| minute | `MONITORING` | `CheckoutObserver::fetch` | `subdomains/core/monitoring/observers/checkout.observer.ts` | -| minute | `PAY_IN` | `CitreaBaseStrategy::checkPayInEntries` | `subdomains/supporting/payin/strategies/register/impl/base/citrea.strategy.ts` | -| minute | `CUSTODY` | `CustodyJobService::handleOrders` | `subdomains/core/custody/services/custody-job.service.ts` | -| minute | `LATEST_BALANCE_CACHE` | `DashboardFinancialService::refreshLatestBalance` | `subdomains/supporting/dashboard/dashboard-financial.service.ts` | -| minute | `FIAT_OUTPUT` | `FiatOutputJobService::fillFiatOutput` | `subdomains/supporting/fiat-output/fiat-output-job.service.ts` | -| minute | `FIAT_PAY_IN` | `FiatPayInSyncService::syncCheckout` | `subdomains/supporting/fiat-payin/services/fiat-payin-sync.service.ts` | -| minute | `PAY_IN` | `InternetComputerStrategy::checkPayInEntries` | `subdomains/supporting/payin/strategies/register/impl/icp.strategy.ts` | -| minute | `JWT_REVOCATION_SYNC` | `JwtRevocationSyncService::syncDeniedJwtAccounts` | `subdomains/generic/user/models/user-data/jwt-revocation-sync.service.ts` | -| minute | `KYC` | `KycService::reviewKycSteps` | `subdomains/generic/kyc/services/kyc.service.ts` | -| minute | `LEDGER_BOOKING_BANK_TX` | `LedgerBookingJobService::runBankTx` | `subdomains/core/accounting/services/ledger-booking-job.service.ts` | -| minute | `LEDGER_BOOKING_BUY_CRYPTO` | `LedgerBookingJobService::runBuyCrypto` | `subdomains/core/accounting/services/ledger-booking-job.service.ts` | -| minute | `LEDGER_BOOKING_BUY_FIAT` | `LedgerBookingJobService::runBuyFiat` | `subdomains/core/accounting/services/ledger-booking-job.service.ts` | -| minute | `LEDGER_BOOKING_CRYPTO_INPUT` | `LedgerBookingJobService::runCryptoInput` | `subdomains/core/accounting/services/ledger-booking-job.service.ts` | -| minute | `LEDGER_BOOKING_EXCHANGE_TX` | `LedgerBookingJobService::runExchangeTx` | `subdomains/core/accounting/services/ledger-booking-job.service.ts` | -| minute | `LEDGER_BOOKING_LIQUIDITY_MANAGEMENT` | `LedgerBookingJobService::runLiquidityMgmt` | `subdomains/core/accounting/services/ledger-booking-job.service.ts` | -| minute | `LEDGER_BOOKING_LIQUIDITY_ORDER` | `LedgerBookingJobService::runLiquidityOrderDex` | `subdomains/core/accounting/services/ledger-booking-job.service.ts` | -| minute | `LEDGER_BOOKING_PAYOUT` | `LedgerBookingJobService::runPayoutOrder` | `subdomains/core/accounting/services/ledger-booking-job.service.ts` | -| minute | `LEDGER_BOOKING_TRADING_ORDER` | `LedgerBookingJobService::runTradingOrder` | `subdomains/core/accounting/services/ledger-booking-job.service.ts` | -| minute | `LIQUIDITY_MANAGEMENT_CHECK_BALANCES` | `LiquidityManagementService::checkLiquidityBalances` | `subdomains/core/liquidity-management/services/liquidity-management.service.ts` | -| minute | `MONITORING` | `LiquidityObserver::fetch` | `subdomains/core/monitoring/observers/liquidity.observer.ts` | -| minute | `TRADING_LOG` | `LogJobService::saveTradingLog` | `subdomains/supporting/log/log-job.service.ts` | -| minute | `MONITORING` | `NodeHealthObserver::fetch` | `subdomains/core/monitoring/observers/node-health.observer.ts` | -| minute | `ORGANIZATION_SYNC` | `OrganizationService::syncOrganization` | `subdomains/generic/user/models/organization/organization.service.ts` | -| minute | `PAY_IN` | `PayInService::checkConfirmations` | `subdomains/supporting/payin/services/payin.service.ts` | -| minute | `PAY_IN` | `PayInService::forwardPayInEntries` | `subdomains/supporting/payin/services/payin.service.ts` | -| minute | `PAY_IN` | `PayInService::returnPayInEntries` | `subdomains/supporting/payin/services/payin.service.ts` | -| minute | `PAYMENT_CONFIRMATIONS` | `PaymentCronService::checkTxConfirmations` | `subdomains/core/payment-link/services/payment-cron.service.ts` | -| minute | `PAYMENT_EXPIRATION` | `PaymentCronService::processExpiredPayments` | `subdomains/core/payment-link/services/payment-cron.service.ts` | -| minute | `UPDATE_BLOCKCHAIN_FEE` | `PaymentLinkFeeService::updateFees` | `subdomains/core/payment-link/services/payment-link-fee.service.ts` | -| minute | `REALUNIT_QUOTE_COMPLETION` | `RealUnitJobService::completeSettledQuotes` | `subdomains/supporting/realunit/realunit-job.service.ts` | -| minute | `SUPPORT_BOT` | `SupportIssueJobService::sendAutoResponses` | `subdomains/supporting/support-issue/services/support-issue-job.service.ts` | -| minute | `TFA_CACHE` | `TfaService::processCleanupSecretCache` | `subdomains/generic/kyc/services/tfa.service.ts` | -| minute | `TRADING` | `TradingJobService::processOrders` | `subdomains/core/trading/services/trading-job.service.ts` | -| minute | `TRADING` | `TradingJobService::processRules` | `subdomains/core/trading/services/trading-job.service.ts` | -| minute | — | `TransactionController::checkLists` | `subdomains/core/history/controllers/transaction.controller.ts` | -| minute | `TX_MAIL` | `TransactionNotificationService::sendNotificationMails` | `subdomains/supporting/payment/services/transaction-notification.service.ts` | -| minute | `TX_REQUEST` | `TransactionRequestService::txRequestStatusSync` | `subdomains/supporting/payment/services/transaction-request.service.ts` | -| minute | `USER_DATA` | `UserDataJobService::fillUserData` | `subdomains/generic/user/models/user-data/user-data-job.service.ts` | -| minute | — | `UserDataService::processCleanupMailSecretCache` | `subdomains/generic/user/models/user-data/user-data.service.ts` | -| minute | `USER` | `UserJobService::fillUser` | `subdomains/generic/user/models/user/user-job.service.ts` | -| 5 minutes | `PRICING` | `AssetPricesJobService::updatePaymentPrices` | `subdomains/supporting/pricing/services/asset-prices-job.service.ts` | -| 5 minutes | `LNURL_AUTH_CACHE` | `AuthLnUrlService::processCleanupAuthCache` | `subdomains/generic/user/models/auth/auth-lnurl.service.ts` | -| 5 minutes | `BANK_TX_RETURN` | `BankTxReturnService::fillBankTxReturn` | `subdomains/supporting/bank-tx/bank-tx-return/bank-tx-return.service.ts` | -| 5 minutes | `BANK_TX` | `BankTxService::enrichYapealTransactions` | `subdomains/supporting/bank-tx/bank-tx/services/bank-tx.service.ts` | -| 5 minutes | `BLOCKCHAIN_CONFIG_CHECK` | `BlockchainConfigCheckService::logUnconfiguredClients` | `integration/blockchain/shared/services/blockchain-config-check.service.ts` | -| 5 minutes | `EXCHANGE_TX_SYNC` | `ExchangeTxService::syncExchangeJob` | `integration/exchange/services/exchange-tx.service.ts` | -| 5 minutes | `CRYPTO_PAYOUT` | `FaucetRequestService::checkFaucetRequests` | `subdomains/core/faucet-request/services/faucet-request.service.ts` | -| 5 minutes | `LEDGER_COA_BOOTSTRAP` | `LedgerBookingJobService::runCoaBootstrap` | `subdomains/core/accounting/services/ledger-booking-job.service.ts` | -| 5 minutes | `LEDGER_CUTOVER` | `LedgerCutoverService::run` | `subdomains/core/accounting/services/ledger-cutover.service.ts` | -| 5 minutes | `LIMIT_REQUEST_MAIL` | `LimitRequestNotificationService::sendNotificationMails` | `subdomains/supporting/support-issue/services/limit-request-notification.service.ts` | -| 5 minutes | `LIQUIDITY_MANAGEMENT` | `LiquidityManagementRuleService::reactivateRules` | `subdomains/core/liquidity-management/services/liquidity-management-rule.service.ts` | -| 5 minutes | `PAY_IN_MAIL` | `PayInNotificationService::sendNotificationMails` | `subdomains/supporting/payin/services/payin-notification.service.ts` | -| 5 minutes | `REALUNIT_TRANSFER_RECONCILIATION` | `RealUnitJobService::reconcilePendingTransfers` | `subdomains/supporting/realunit/realunit-job.service.ts` | -| 5 minutes | `SUPPORT_BOT` | `SupportEscalationService::checkEscalations` | `subdomains/supporting/support-issue/services/support-escalation.service.ts` | -| 5 minutes | `TRADING` | `TradingJobService::reactivateRules` | `subdomains/core/trading/services/trading-job.service.ts` | -| 5 minutes | — | `TransactionHelper::updateCache` | `subdomains/supporting/payment/services/transaction-helper.ts` | -| 5 minutes | `WEBHOOK` | `WebhookNotificationService::sendWebhooks` | `subdomains/generic/user/services/webhook/webhook-notification.service.ts` | -| 10 minutes | `BANK_ACCOUNT` | `BankAccountService::reloadUncheckedBankAccounts` | `subdomains/supporting/bank/bank-account/bank-account.service.ts` | -| 10 minutes | `DEURO_LOG_INFO` | `DEuroService::processLogInfo` | `integration/blockchain/deuro/deuro.service.ts` | -| 10 minutes | `MONITORING` | `ExchangeObserver::fetch` | `subdomains/core/monitoring/observers/exchange.observer.ts` | -| 10 minutes | `MONITORING` | `ExternalServicesObserver::fetch` | `subdomains/core/monitoring/observers/external-services.observer.ts` | -| 10 minutes | `BLOCKCHAIN_FEE_UPDATE` | `FeeService::updateBlockchainFees` | `subdomains/supporting/payment/services/fee.service.ts` | -| 10 minutes | `FRANKENCOIN_LOG_INFO` | `FrankencoinService::processLogInfo` | `integration/blockchain/frankencoin/frankencoin.service.ts` | -| 10 minutes | `JUICE_LOG_INFO` | `JuiceService::processLogInfo` | `integration/blockchain/juice/juice.service.ts` | -| 10 minutes | `MONITORING` | `NodeBalanceObserver::fetch` | `subdomains/core/monitoring/observers/node-balance.observer.ts` | -| 10 minutes | `MAIL_RETRY` | `NotificationJobService::resendUncompletedMails` | `subdomains/supporting/notification/services/notification-job.service.ts` | -| 10 minutes | `PAY_IN` | `PayInService::updateFailedPayments` | `subdomains/supporting/payin/services/payin.service.ts` | -| 10 minutes | `MONITORING` | `PaymentObserver::fetch` | `subdomains/core/monitoring/observers/payment.observer.ts` | -| 10 minutes | `MONITORING` | `RealUnitW2wGasObserver::fetch` | `subdomains/core/monitoring/observers/realunit-w2w-gas.observer.ts` | -| 10 minutes | `REF_PAYOUT` | `RefRewardJobService::processPendingRefRewards` | `subdomains/core/referral/reward/services/ref-reward-job.service.ts` | -| 10 minutes | `MONITORING` | `UserObserver::fetch` | `subdomains/core/monitoring/observers/user.observer.ts` | -| 10 minutes | `ZANO_ASSET_WHITELIST` | `ZanoService::setupAssetWhitelist` | `integration/blockchain/zano/services/zano.service.ts` | -| hour | `PRICING` | `AssetPricesJobService::updatePrices` | `subdomains/supporting/pricing/services/asset-prices-job.service.ts` | -| hour | `BANK_ACCOUNT` | `BankAccountService::reloadErrorBankAccounts` | `subdomains/supporting/bank/bank-account/bank-account.service.ts` | -| hour | `BINANCE_PAY_CERTIFICATES_UPDATE` | `BinancePayService::updateCertificates` | `integration/binance-pay/services/binance-pay.service.ts` | -| hour | `BUY_CRYPTO_AGGREGATION` | `BuyCryptoJobService::checkAggregatingTransactions` | `subdomains/core/buy-crypto/process/services/buy-crypto-job.service.ts` | -| hour | `ASSET_DECIMALS` | `EvmDecimalsService::setDecimals` | `integration/blockchain/shared/evm/evm-decimals.service.ts` | -| hour | `FIAT_OUTPUT` | `FiatOutputFrickService::checkFrickOrderStatus` | `subdomains/supporting/fiat-output/fiat-output-frick.service.ts` | -| hour | `FIAT_OUTPUT` | `FiatOutputJobService::checkOlkypayOrderStatus` | `subdomains/supporting/fiat-output/fiat-output-job.service.ts` | -| hour | `FIAT_OUTPUT` | `FiatOutputJobService::generateReports` | `subdomains/supporting/fiat-output/fiat-output-job.service.ts` | -| hour | `PRICING` | `FiatPricesService::updatePrices` | `subdomains/supporting/pricing/services/fiat-prices.service.ts` | -| hour | `KYC_MAIL` | `KycNotificationService::sendNotificationMails` | `subdomains/generic/kyc/services/kyc-notification.service.ts` | -| hour | `PAYMENT_FORWARDING` | `PaymentCronService::forwardDeposits` | `subdomains/core/payment-link/services/payment-cron.service.ts` | -| hour | `REF_CLEANUP` | `RefService::checkRefs` | `subdomains/core/referral/process/ref.service.ts` | -| hour | `UPDATE_STATISTIC` | `StatisticService::doUpdate` | `subdomains/core/statistic/statistic.service.ts` | -| hour | `SUPPORT_BOT` | `SupportIssueJobService::autoOnHold` | `subdomains/supporting/support-issue/services/support-issue-job.service.ts` | -| hour | `BLACK_SQUAD_MAIL` | `UserDataNotificationService::sendNotificationMails` | `subdomains/generic/user/models/user-data/user-data-notification.service.ts` | -| hour | `VIRTUAL_IBAN_FRICK_ISSUANCE_RECONCILIATION` | `VirtualIbanFrickIssuanceReconciliationService::reconcileRetiredIssuanceReferences` | `subdomains/supporting/bank/virtual-iban/virtual-iban-frick-issuance-reconciliation.service.ts` | -| day at 3am | `TX_REQUEST_WAITING_EXPIRY` | `TransactionRequestService::txRequestWaitingExpiryCheck` | `subdomains/supporting/payment/services/transaction-request.service.ts` | -| day at 4am | `CUSTODY` | `CustodyJobService::resetExpiredConfirmedOrders` | `subdomains/core/custody/services/custody-job.service.ts` | -| day at 4am | `KYC` | `KycService::checkIdentSteps` | `subdomains/generic/kyc/services/kyc.service.ts` | -| day at 4am | `LEDGER_MARK_TO_MARKET` | `LedgerMarkToMarketService::run` | `subdomains/core/accounting/services/ledger-mark-to-market.service.ts` | -| day at 5am | `LEDGER_RECONCILIATION` | `LedgerReconciliationService::run` | `subdomains/core/accounting/services/ledger-reconciliation.service.ts` | -| day at 6am | `REF_PAYOUT` | `RefRewardJobService::createPendingRefRewards` | `subdomains/core/referral/reward/services/ref-reward-job.service.ts` | -| day at 11pm | `LOG_CLEANUP` | `LogService::cleanup` | `subdomains/supporting/log/log.service.ts` | -| week | `BANK_ACCOUNT` | `BankAccountService::checkFailedBankAccounts` | `subdomains/supporting/bank/bank-account/bank-account.service.ts` | -| weekend | `SANCTION_SYNC` | `SanctionService::syncList` | `subdomains/core/aml/services/sanction.service.ts` | -| 1st day of month at midnight | — | `BuyService::resetMonthlyVolumes` | `subdomains/core/buy-crypto/routes/buy/buy.service.ts` | -| 1st day of month at midnight | — | `SellService::resetMonthlyVolumes` | `subdomains/core/sell-crypto/route/sell.service.ts` | -| 1st day of month at midnight | — | `SwapService::resetMonthlyVolumes` | `subdomains/core/buy-crypto/routes/swap/swap.service.ts` | -| 1st day of month at midnight | — | `UserDataService::resetMonthlyVolumes` | `subdomains/generic/user/models/user-data/user-data.service.ts` | -| 1st day of month at midnight | — | `UserService::resetMonthlyVolumes` | `subdomains/generic/user/models/user/user.service.ts` | -| year | — | `BuyService::resetAnnualVolumes` | `subdomains/core/buy-crypto/routes/buy/buy.service.ts` | -| year | — | `SellService::resetAnnualVolumes` | `subdomains/core/sell-crypto/route/sell.service.ts` | -| year | — | `SwapService::resetAnnualVolumes` | `subdomains/core/buy-crypto/routes/swap/swap.service.ts` | -| year | — | `UserDataService::resetAnnualVolumes` | `subdomains/generic/user/models/user-data/user-data.service.ts` | -| year | — | `UserService::resetAnnualVolumes` | `subdomains/generic/user/models/user/user.service.ts` | +| Interval | Flag | Scope | Job | File | +| -------- | ---- | ----- | --- | ---- | +| second | `PAY_IN` | `worker` | `BitcoinStrategy::checkPayInEntries` | `subdomains/supporting/payin/strategies/register/impl/bitcoin.strategy.ts` | +| second | `PAY_IN` | `worker` | `FiroStrategy::checkPayInEntries` | `subdomains/supporting/payin/strategies/register/impl/firo.strategy.ts` | +| second | `PAY_IN` | `worker` | `MoneroStrategy::checkPayInEntries` | `subdomains/supporting/payin/strategies/register/impl/monero.strategy.ts` | +| second | `MONITOR_CONNECTION_POOL` | `both` | `MonitorConnectionPoolService::monitorConnectionPool` | `subdomains/core/monitoring/monitor-connection-pool.service.ts` | +| second | `PAY_IN` | `worker` | `ZanoStrategy::checkPayInEntries` | `subdomains/supporting/payin/strategies/register/impl/zano.strategy.ts` | +| 10 seconds | `LIQUIDITY_MANAGEMENT` | `worker` | `LiquidityManagementPipelineService::processPipelines` | `subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.ts` | +| 10 seconds | `MONITOR_CONNECTION_POOL` | `both` | `MonitorConnectionPoolService::monitorConnectionPoolStatic` | `subdomains/core/monitoring/monitor-connection-pool.service.ts` | +| 10 seconds | `MONITOR_EVENT_LOOP` | `both` | `MonitorEventLoopService::monitorEventLoop` | `subdomains/core/monitoring/monitor-event-loop.service.ts` | +| 30 seconds | `LNURL_AUTH_CACHE` | `both` | `AuthLnUrlService::processCleanupAccessToken` | `subdomains/generic/user/models/auth/auth-lnurl.service.ts` | +| 30 seconds | `BANK_TX` | `worker` | `BankTxService::checkBankTx` | `subdomains/supporting/bank-tx/bank-tx/services/bank-tx.service.ts` | +| 30 seconds | `DEX_PURCHASE_ORDER` | `worker` | `DexService::finalizePurchaseOrders` | `subdomains/supporting/dex/services/dex.service.ts` | +| 30 seconds | — | `api` | `ExchangeController::checkTrades` | `integration/exchange/controllers/exchange.controller.ts` | +| 30 seconds | `PAY_OUT` | `worker` | `PayoutService::processOrders` | `subdomains/supporting/payout/services/payout.service.ts` | +| 30 seconds | — | `both` | `ProcessService::resyncDeniedJwtAccounts` | `shared/services/process.service.ts` | +| 30 seconds | — | `both` | `ProcessService::resyncDeniedJwtAddresses` | `shared/services/process.service.ts` | +| 30 seconds | — | `both` | `ProcessService::resyncDisabledProcesses` | `shared/services/process.service.ts` | +| minute | `PAY_OUT` | `worker` | `AdminService::completeLiquidityOrders` | `subdomains/generic/admin/admin.service.ts` | +| minute | `MONITORING` | `worker` | `AmlObserver::fetch` | `subdomains/core/monitoring/observers/aml.observer.ts` | +| minute | — | `both` | `AuthService::checkLists` | `subdomains/generic/user/models/auth/auth.service.ts` | +| minute | `BANK_DATA_VERIFICATION` | `worker` | `BankDataService::checkAndSetActive` | `subdomains/generic/user/models/bank-data/bank-data.service.ts` | +| minute | `MONITORING` | `worker` | `BankObserver::fetch` | `subdomains/core/monitoring/observers/bank.observer.ts` | +| minute | `BANK_TX_RETURN_MAIL` | `worker` | `BankTxReturnNotificationService::sendBankTxReturnMail` | `subdomains/supporting/bank-tx/bank-tx-return/bank-tx-return-notification.service.ts` | +| minute | `BUY_CRYPTO` | `worker` | `BuyCryptoJobService::process` | `subdomains/core/buy-crypto/process/services/buy-crypto-job.service.ts` | +| minute | `BUY_FIAT` | `worker` | `BuyFiatJobService::addFiatOutputs` | `subdomains/core/sell-crypto/process/services/buy-fiat-job.service.ts` | +| minute | `BUY_FIAT` | `worker` | `BuyFiatJobService::checkCryptoPayIn` | `subdomains/core/sell-crypto/process/services/buy-fiat-job.service.ts` | +| minute | `BUY_FIAT_MAIL` | `worker` | `BuyFiatNotificationService::sendNotificationMails` | `subdomains/core/sell-crypto/process/services/buy-fiat-notification.service.ts` | +| minute | `PAY_IN` | `worker` | `CardanoStrategy::checkPayInEntries` | `subdomains/supporting/payin/strategies/register/impl/cardano.strategy.ts` | +| minute | `MONITORING` | `worker` | `CheckoutObserver::fetch` | `subdomains/core/monitoring/observers/checkout.observer.ts` | +| minute | `PAY_IN` | `worker` | `CitreaBaseStrategy::checkPayInEntries` | `subdomains/supporting/payin/strategies/register/impl/base/citrea.strategy.ts` | +| minute | `CUSTODY` | `worker` | `CustodyJobService::handleOrders` | `subdomains/core/custody/services/custody-job.service.ts` | +| minute | `LATEST_BALANCE_CACHE` | `api` | `DashboardFinancialService::refreshLatestBalance` | `subdomains/supporting/dashboard/dashboard-financial.service.ts` | +| minute | `FIAT_OUTPUT` | `worker` | `FiatOutputJobService::fillFiatOutput` | `subdomains/supporting/fiat-output/fiat-output-job.service.ts` | +| minute | `FIAT_PAY_IN` | `worker` | `FiatPayInSyncService::syncCheckout` | `subdomains/supporting/fiat-payin/services/fiat-payin-sync.service.ts` | +| minute | `PAY_IN` | `worker` | `InternetComputerStrategy::checkPayInEntries` | `subdomains/supporting/payin/strategies/register/impl/icp.strategy.ts` | +| minute | `JWT_REVOCATION_SYNC` | `worker` | `JwtRevocationSyncService::syncDeniedJwtAccounts` | `subdomains/generic/user/models/user-data/jwt-revocation-sync.service.ts` | +| minute | `KYC` | `worker` | `KycService::reviewKycSteps` | `subdomains/generic/kyc/services/kyc.service.ts` | +| minute | `LEDGER_BOOKING_BANK_TX` | `worker` | `LedgerBookingJobService::runBankTx` | `subdomains/core/accounting/services/ledger-booking-job.service.ts` | +| minute | `LEDGER_BOOKING_BUY_CRYPTO` | `worker` | `LedgerBookingJobService::runBuyCrypto` | `subdomains/core/accounting/services/ledger-booking-job.service.ts` | +| minute | `LEDGER_BOOKING_BUY_FIAT` | `worker` | `LedgerBookingJobService::runBuyFiat` | `subdomains/core/accounting/services/ledger-booking-job.service.ts` | +| minute | `LEDGER_BOOKING_CRYPTO_INPUT` | `worker` | `LedgerBookingJobService::runCryptoInput` | `subdomains/core/accounting/services/ledger-booking-job.service.ts` | +| minute | `LEDGER_BOOKING_EXCHANGE_TX` | `worker` | `LedgerBookingJobService::runExchangeTx` | `subdomains/core/accounting/services/ledger-booking-job.service.ts` | +| minute | `LEDGER_BOOKING_LIQUIDITY_MANAGEMENT` | `worker` | `LedgerBookingJobService::runLiquidityMgmt` | `subdomains/core/accounting/services/ledger-booking-job.service.ts` | +| minute | `LEDGER_BOOKING_LIQUIDITY_ORDER` | `worker` | `LedgerBookingJobService::runLiquidityOrderDex` | `subdomains/core/accounting/services/ledger-booking-job.service.ts` | +| minute | `LEDGER_BOOKING_PAYOUT` | `worker` | `LedgerBookingJobService::runPayoutOrder` | `subdomains/core/accounting/services/ledger-booking-job.service.ts` | +| minute | `LEDGER_BOOKING_TRADING_ORDER` | `worker` | `LedgerBookingJobService::runTradingOrder` | `subdomains/core/accounting/services/ledger-booking-job.service.ts` | +| minute | `LIQUIDITY_MANAGEMENT_CHECK_BALANCES` | `worker` | `LiquidityManagementService::checkLiquidityBalances` | `subdomains/core/liquidity-management/services/liquidity-management.service.ts` | +| minute | `MONITORING` | `worker` | `LiquidityObserver::fetch` | `subdomains/core/monitoring/observers/liquidity.observer.ts` | +| minute | `TRADING_LOG` | `worker` | `LogJobService::saveTradingLog` | `subdomains/supporting/log/log-job.service.ts` | +| minute | `MONITORING` | `worker` | `NodeHealthObserver::fetch` | `subdomains/core/monitoring/observers/node-health.observer.ts` | +| minute | `ORGANIZATION_SYNC` | `worker` | `OrganizationService::syncOrganization` | `subdomains/generic/user/models/organization/organization.service.ts` | +| minute | `PAY_IN` | `worker` | `PayInService::checkConfirmations` | `subdomains/supporting/payin/services/payin.service.ts` | +| minute | `PAY_IN` | `worker` | `PayInService::forwardPayInEntries` | `subdomains/supporting/payin/services/payin.service.ts` | +| minute | `PAY_IN` | `worker` | `PayInService::returnPayInEntries` | `subdomains/supporting/payin/services/payin.service.ts` | +| minute | `PAYMENT_CONFIRMATIONS` | `api` | `PaymentCronService::checkTxConfirmations` | `subdomains/core/payment-link/services/payment-cron.service.ts` | +| minute | `PAYMENT_EXPIRATION` | `api` | `PaymentCronService::processExpiredPayments` | `subdomains/core/payment-link/services/payment-cron.service.ts` | +| minute | `UPDATE_BLOCKCHAIN_FEE` | `both` | `PaymentLinkFeeService::updateFees` | `subdomains/core/payment-link/services/payment-link-fee.service.ts` | +| minute | `REALUNIT_QUOTE_COMPLETION` | `worker` | `RealUnitJobService::completeSettledQuotes` | `subdomains/supporting/realunit/realunit-job.service.ts` | +| minute | `SUPPORT_BOT` | `worker` | `SupportIssueJobService::sendAutoResponses` | `subdomains/supporting/support-issue/services/support-issue-job.service.ts` | +| minute | `TFA_CACHE` | `both` | `TfaService::processCleanupSecretCache` | `subdomains/generic/kyc/services/tfa.service.ts` | +| minute | `TRADING` | `worker` | `TradingJobService::processOrders` | `subdomains/core/trading/services/trading-job.service.ts` | +| minute | `TRADING` | `worker` | `TradingJobService::processRules` | `subdomains/core/trading/services/trading-job.service.ts` | +| minute | — | `api` | `TransactionController::checkLists` | `subdomains/core/history/controllers/transaction.controller.ts` | +| minute | `TX_MAIL` | `worker` | `TransactionNotificationService::sendNotificationMails` | `subdomains/supporting/payment/services/transaction-notification.service.ts` | +| minute | `TX_REQUEST` | `worker` | `TransactionRequestService::txRequestStatusSync` | `subdomains/supporting/payment/services/transaction-request.service.ts` | +| minute | `USER_DATA` | `worker` | `UserDataJobService::fillUserData` | `subdomains/generic/user/models/user-data/user-data-job.service.ts` | +| minute | — | `both` | `UserDataService::processCleanupMailSecretCache` | `subdomains/generic/user/models/user-data/user-data.service.ts` | +| minute | `USER` | `worker` | `UserJobService::fillUser` | `subdomains/generic/user/models/user/user-job.service.ts` | +| 5 minutes | `PRICING` | `worker` | `AssetPricesJobService::updatePaymentPrices` | `subdomains/supporting/pricing/services/asset-prices-job.service.ts` | +| 5 minutes | `LNURL_AUTH_CACHE` | `both` | `AuthLnUrlService::processCleanupAuthCache` | `subdomains/generic/user/models/auth/auth-lnurl.service.ts` | +| 5 minutes | `BANK_TX_RETURN` | `worker` | `BankTxReturnService::fillBankTxReturn` | `subdomains/supporting/bank-tx/bank-tx-return/bank-tx-return.service.ts` | +| 5 minutes | `BANK_TX` | `worker` | `BankTxService::enrichYapealTransactions` | `subdomains/supporting/bank-tx/bank-tx/services/bank-tx.service.ts` | +| 5 minutes | `BLOCKCHAIN_CONFIG_CHECK` | `worker` | `BlockchainConfigCheckService::logUnconfiguredClients` | `integration/blockchain/shared/services/blockchain-config-check.service.ts` | +| 5 minutes | `EXCHANGE_TX_SYNC` | `worker` | `ExchangeTxService::syncExchangeJob` | `integration/exchange/services/exchange-tx.service.ts` | +| 5 minutes | `CRYPTO_PAYOUT` | `worker` | `FaucetRequestService::checkFaucetRequests` | `subdomains/core/faucet-request/services/faucet-request.service.ts` | +| 5 minutes | `LEDGER_COA_BOOTSTRAP` | `worker` | `LedgerBookingJobService::runCoaBootstrap` | `subdomains/core/accounting/services/ledger-booking-job.service.ts` | +| 5 minutes | `LEDGER_CUTOVER` | `worker` | `LedgerCutoverService::run` | `subdomains/core/accounting/services/ledger-cutover.service.ts` | +| 5 minutes | `LIMIT_REQUEST_MAIL` | `worker` | `LimitRequestNotificationService::sendNotificationMails` | `subdomains/supporting/support-issue/services/limit-request-notification.service.ts` | +| 5 minutes | `LIQUIDITY_MANAGEMENT` | `worker` | `LiquidityManagementRuleService::reactivateRules` | `subdomains/core/liquidity-management/services/liquidity-management-rule.service.ts` | +| 5 minutes | `PAY_IN_MAIL` | `worker` | `PayInNotificationService::sendNotificationMails` | `subdomains/supporting/payin/services/payin-notification.service.ts` | +| 5 minutes | `REALUNIT_TRANSFER_RECONCILIATION` | `worker` | `RealUnitJobService::reconcilePendingTransfers` | `subdomains/supporting/realunit/realunit-job.service.ts` | +| 5 minutes | `SUPPORT_BOT` | `worker` | `SupportEscalationService::checkEscalations` | `subdomains/supporting/support-issue/services/support-escalation.service.ts` | +| 5 minutes | `TRADING` | `worker` | `TradingJobService::reactivateRules` | `subdomains/core/trading/services/trading-job.service.ts` | +| 5 minutes | — | `both` | `TransactionHelper::updateCache` | `subdomains/supporting/payment/services/transaction-helper.ts` | +| 5 minutes | `WEBHOOK` | `worker` | `WebhookNotificationService::sendWebhooks` | `subdomains/generic/user/services/webhook/webhook-notification.service.ts` | +| 10 minutes | `BANK_ACCOUNT` | `worker` | `BankAccountService::reloadUncheckedBankAccounts` | `subdomains/supporting/bank/bank-account/bank-account.service.ts` | +| 10 minutes | `DEURO_LOG_INFO` | `worker` | `DEuroService::processLogInfo` | `integration/blockchain/deuro/deuro.service.ts` | +| 10 minutes | `MONITORING` | `worker` | `ExchangeObserver::fetch` | `subdomains/core/monitoring/observers/exchange.observer.ts` | +| 10 minutes | `MONITORING` | `worker` | `ExternalServicesObserver::fetch` | `subdomains/core/monitoring/observers/external-services.observer.ts` | +| 10 minutes | `BLOCKCHAIN_FEE_UPDATE` | `worker` | `FeeService::updateBlockchainFees` | `subdomains/supporting/payment/services/fee.service.ts` | +| 10 minutes | `FRANKENCOIN_LOG_INFO` | `worker` | `FrankencoinService::processLogInfo` | `integration/blockchain/frankencoin/frankencoin.service.ts` | +| 10 minutes | `JUICE_LOG_INFO` | `worker` | `JuiceService::processLogInfo` | `integration/blockchain/juice/juice.service.ts` | +| 10 minutes | `MONITORING` | `worker` | `NodeBalanceObserver::fetch` | `subdomains/core/monitoring/observers/node-balance.observer.ts` | +| 10 minutes | `MAIL_RETRY` | `worker` | `NotificationJobService::resendUncompletedMails` | `subdomains/supporting/notification/services/notification-job.service.ts` | +| 10 minutes | `PAY_IN` | `worker` | `PayInService::updateFailedPayments` | `subdomains/supporting/payin/services/payin.service.ts` | +| 10 minutes | `MONITORING` | `worker` | `PaymentObserver::fetch` | `subdomains/core/monitoring/observers/payment.observer.ts` | +| 10 minutes | `MONITORING` | `worker` | `RealUnitW2wGasObserver::fetch` | `subdomains/core/monitoring/observers/realunit-w2w-gas.observer.ts` | +| 10 minutes | `REF_PAYOUT` | `worker` | `RefRewardJobService::processPendingRefRewards` | `subdomains/core/referral/reward/services/ref-reward-job.service.ts` | +| 10 minutes | `MONITORING` | `worker` | `UserObserver::fetch` | `subdomains/core/monitoring/observers/user.observer.ts` | +| 10 minutes | `ZANO_ASSET_WHITELIST` | `worker` | `ZanoService::setupAssetWhitelist` | `integration/blockchain/zano/services/zano.service.ts` | +| hour | `PRICING` | `worker` | `AssetPricesJobService::updatePrices` | `subdomains/supporting/pricing/services/asset-prices-job.service.ts` | +| hour | `BANK_ACCOUNT` | `worker` | `BankAccountService::reloadErrorBankAccounts` | `subdomains/supporting/bank/bank-account/bank-account.service.ts` | +| hour | `BINANCE_PAY_CERTIFICATES_UPDATE` | `worker` | `BinancePayService::updateCertificates` | `integration/binance-pay/services/binance-pay.service.ts` | +| hour | `BUY_CRYPTO_AGGREGATION` | `worker` | `BuyCryptoJobService::checkAggregatingTransactions` | `subdomains/core/buy-crypto/process/services/buy-crypto-job.service.ts` | +| hour | `ASSET_DECIMALS` | `worker` | `EvmDecimalsService::setDecimals` | `integration/blockchain/shared/evm/evm-decimals.service.ts` | +| hour | `FIAT_OUTPUT` | `worker` | `FiatOutputFrickService::checkFrickOrderStatus` | `subdomains/supporting/fiat-output/fiat-output-frick.service.ts` | +| hour | `FIAT_OUTPUT` | `worker` | `FiatOutputJobService::checkOlkypayOrderStatus` | `subdomains/supporting/fiat-output/fiat-output-job.service.ts` | +| hour | `FIAT_OUTPUT` | `worker` | `FiatOutputJobService::generateReports` | `subdomains/supporting/fiat-output/fiat-output-job.service.ts` | +| hour | `PRICING` | `worker` | `FiatPricesService::updatePrices` | `subdomains/supporting/pricing/services/fiat-prices.service.ts` | +| hour | `KYC_MAIL` | `worker` | `KycNotificationService::sendNotificationMails` | `subdomains/generic/kyc/services/kyc-notification.service.ts` | +| hour | `PAYMENT_FORWARDING` | `worker` | `PaymentCronService::forwardDeposits` | `subdomains/core/payment-link/services/payment-cron.service.ts` | +| hour | `REF_CLEANUP` | `worker` | `RefService::checkRefs` | `subdomains/core/referral/process/ref.service.ts` | +| hour | `UPDATE_STATISTIC` | `api` | `StatisticService::doUpdate` | `subdomains/core/statistic/statistic.service.ts` | +| hour | `SUPPORT_BOT` | `worker` | `SupportIssueJobService::autoOnHold` | `subdomains/supporting/support-issue/services/support-issue-job.service.ts` | +| hour | `BLACK_SQUAD_MAIL` | `worker` | `UserDataNotificationService::sendNotificationMails` | `subdomains/generic/user/models/user-data/user-data-notification.service.ts` | +| hour | `VIRTUAL_IBAN_FRICK_ISSUANCE_RECONCILIATION` | `worker` | `VirtualIbanFrickIssuanceReconciliationService::reconcileRetiredIssuanceReferences` | `subdomains/supporting/bank/virtual-iban/virtual-iban-frick-issuance-reconciliation.service.ts` | +| day at 3am | `TX_REQUEST_WAITING_EXPIRY` | `worker` | `TransactionRequestService::txRequestWaitingExpiryCheck` | `subdomains/supporting/payment/services/transaction-request.service.ts` | +| day at 4am | `CUSTODY` | `worker` | `CustodyJobService::resetExpiredConfirmedOrders` | `subdomains/core/custody/services/custody-job.service.ts` | +| day at 4am | `KYC` | `worker` | `KycService::checkIdentSteps` | `subdomains/generic/kyc/services/kyc.service.ts` | +| day at 4am | `LEDGER_MARK_TO_MARKET` | `worker` | `LedgerMarkToMarketService::run` | `subdomains/core/accounting/services/ledger-mark-to-market.service.ts` | +| day at 5am | `LEDGER_RECONCILIATION` | `worker` | `LedgerReconciliationService::run` | `subdomains/core/accounting/services/ledger-reconciliation.service.ts` | +| day at 6am | `REF_PAYOUT` | `worker` | `RefRewardJobService::createPendingRefRewards` | `subdomains/core/referral/reward/services/ref-reward-job.service.ts` | +| day at 11pm | `LOG_CLEANUP` | `worker` | `LogService::cleanup` | `subdomains/supporting/log/log.service.ts` | +| week | `BANK_ACCOUNT` | `worker` | `BankAccountService::checkFailedBankAccounts` | `subdomains/supporting/bank/bank-account/bank-account.service.ts` | +| weekend | `SANCTION_SYNC` | `worker` | `SanctionService::syncList` | `subdomains/core/aml/services/sanction.service.ts` | +| 1st day of month at midnight | — | `worker` | `BuyService::resetMonthlyVolumes` | `subdomains/core/buy-crypto/routes/buy/buy.service.ts` | +| 1st day of month at midnight | — | `worker` | `SellService::resetMonthlyVolumes` | `subdomains/core/sell-crypto/route/sell.service.ts` | +| 1st day of month at midnight | — | `worker` | `SwapService::resetMonthlyVolumes` | `subdomains/core/buy-crypto/routes/swap/swap.service.ts` | +| 1st day of month at midnight | — | `worker` | `UserDataService::resetMonthlyVolumes` | `subdomains/generic/user/models/user-data/user-data.service.ts` | +| 1st day of month at midnight | — | `worker` | `UserService::resetMonthlyVolumes` | `subdomains/generic/user/models/user/user.service.ts` | +| year | — | `worker` | `BuyService::resetAnnualVolumes` | `subdomains/core/buy-crypto/routes/buy/buy.service.ts` | +| year | — | `worker` | `SellService::resetAnnualVolumes` | `subdomains/core/sell-crypto/route/sell.service.ts` | +| year | — | `worker` | `SwapService::resetAnnualVolumes` | `subdomains/core/buy-crypto/routes/swap/swap.service.ts` | +| year | — | `worker` | `UserDataService::resetAnnualVolumes` | `subdomains/generic/user/models/user-data/user-data.service.ts` | +| year | — | `worker` | `UserService::resetAnnualVolumes` | `subdomains/generic/user/models/user/user.service.ts` | diff --git a/src/__tests__/tracing.spec.ts b/src/__tests__/tracing.spec.ts index 54b796b101..7c984cbcb0 100644 --- a/src/__tests__/tracing.spec.ts +++ b/src/__tests__/tracing.spec.ts @@ -105,8 +105,8 @@ describe('tracingServiceName', () => { }); it('reports the worker under its own name', () => { - // Both processes run the same image and would otherwise report as one service, mixing the - // worker's outgoing calls into every panel not restricted to server spans. + // Both processes run the same image and would otherwise report as one service, leaving a + // consumer of the traces unable to tell them apart. process.env.CRON_ROLE = 'worker'; expect(tracingServiceName()).toBe('dfx-api-worker'); }); diff --git a/src/config/__tests__/cron-role.config.spec.ts b/src/config/__tests__/cron-role.config.spec.ts index 4bbf6b4198..a4e3c37e95 100644 --- a/src/config/__tests__/cron-role.config.spec.ts +++ b/src/config/__tests__/cron-role.config.spec.ts @@ -2,9 +2,9 @@ import { Config, ConfigService, CronRole, GetConfig, parseCronRole } from '../co describe('parseCronRole', () => { it('accepts the three roles', () => { - expect(parseCronRole('all')).toBe(CronRole.All); - expect(parseCronRole('api')).toBe(CronRole.Api); - expect(parseCronRole('worker')).toBe(CronRole.Worker); + expect(parseCronRole('all')).toBe(CronRole.ALL); + expect(parseCronRole('api')).toBe(CronRole.API); + expect(parseCronRole('worker')).toBe(CronRole.WORKER); }); it.each([undefined, '', ' ', 'All', 'WORKER', 'api ', 'true', 'none'])( @@ -39,9 +39,9 @@ describe('Config.cronRole', () => { // Covers the wiring env -> parseCronRole -> Config that DfxCronService reads, which // unit-testing the parser alone would leave unverified. it.each([ - ['all', CronRole.All], - ['api', CronRole.Api], - ['worker', CronRole.Worker], + ['all', CronRole.ALL], + ['api', CronRole.API], + ['worker', CronRole.WORKER], ])('maps CRON_ROLE=%s to %s', (value, expected) => { process.env.CRON_ROLE = value; diff --git a/src/config/config.ts b/src/config/config.ts index 45af891554..5be64fc114 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -45,11 +45,11 @@ export enum Environment { */ export enum CronRole { /** One process runs everything: local development, tests, and any deployment without a worker. */ - All = 'all', + ALL = 'all', /** Serves HTTP; runs only jobs scoped `api` or `both`. */ - Api = 'api', + API = 'api', /** Runs the background work; only jobs scoped `worker` or `both`. */ - Worker = 'worker', + WORKER = 'worker', } export type StorageWriteMode = 'azure' | 'dual' | 's3'; diff --git a/src/integration/binance-pay/services/binance-pay.service.ts b/src/integration/binance-pay/services/binance-pay.service.ts index bc0fe0127c..5ddee4eea2 100644 --- a/src/integration/binance-pay/services/binance-pay.service.ts +++ b/src/integration/binance-pay/services/binance-pay.service.ts @@ -225,7 +225,7 @@ export class BinancePayService implements C2BPaymentLinkProvider { try { const headers = this.getHeaders({}); diff --git a/src/integration/blockchain/deuro/deuro.service.ts b/src/integration/blockchain/deuro/deuro.service.ts index c815cabb6a..e510138b56 100644 --- a/src/integration/blockchain/deuro/deuro.service.ts +++ b/src/integration/blockchain/deuro/deuro.service.ts @@ -60,7 +60,7 @@ export class DEuroService extends FrankencoinBasedService implements OnModuleIni this.deuroClient = new DEuroClient(this.getEvmClient()); } - @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.DEURO_LOG_INFO }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.WORKER, process: Process.DEURO_LOG_INFO }) async processLogInfo(): Promise { if (!Config.blockchain.deuro.graphUrl || !Config.blockchain.deuro.apiUrl) { this.logger.warn('DEuro graphUrl/apiUrl not configured - skipping processLogInfo'); diff --git a/src/integration/blockchain/frankencoin/frankencoin.service.ts b/src/integration/blockchain/frankencoin/frankencoin.service.ts index 5f12785751..08a38d8e3a 100644 --- a/src/integration/blockchain/frankencoin/frankencoin.service.ts +++ b/src/integration/blockchain/frankencoin/frankencoin.service.ts @@ -51,7 +51,7 @@ export class FrankencoinService extends FrankencoinBasedService implements OnMod this.frankencoinClient = new FrankencoinClient(this.getEvmClient()); } - @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.FRANKENCOIN_LOG_INFO }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.WORKER, process: Process.FRANKENCOIN_LOG_INFO }) async processLogInfo() { if (!Config.blockchain.frankencoin.contractAddress.xchf) { this.logger.warn('Frankencoin xchf contract not configured - skipping processLogInfo'); diff --git a/src/integration/blockchain/juice/juice.service.ts b/src/integration/blockchain/juice/juice.service.ts index 660fe114c9..dc4504270e 100644 --- a/src/integration/blockchain/juice/juice.service.ts +++ b/src/integration/blockchain/juice/juice.service.ts @@ -61,7 +61,7 @@ export class JuiceService extends FrankencoinBasedService implements OnModuleIni return this.registryService.getClient(Blockchain.CITREA) as EvmClient; } - @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.JUICE_LOG_INFO }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.WORKER, process: Process.JUICE_LOG_INFO }) async processLogInfo(): Promise { if (!Config.blockchain.juice.graphUrl || !Config.blockchain.juice.apiUrl) { this.logger.warn('Juice graphUrl/apiUrl not configured - skipping processLogInfo'); diff --git a/src/integration/blockchain/shared/evm/evm-decimals.service.ts b/src/integration/blockchain/shared/evm/evm-decimals.service.ts index 099def2332..cb92762ebf 100644 --- a/src/integration/blockchain/shared/evm/evm-decimals.service.ts +++ b/src/integration/blockchain/shared/evm/evm-decimals.service.ts @@ -19,7 +19,7 @@ export class EvmDecimalsService { ) {} // --- JOBS --- // - @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, process: Process.ASSET_DECIMALS, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.WORKER, process: Process.ASSET_DECIMALS, timeout: 1800 }) async setDecimals() { const assets = await this.assetService.getEvmAssetsWithoutDecimals(EvmBlockchains); diff --git a/src/integration/blockchain/shared/services/blockchain-config-check.service.ts b/src/integration/blockchain/shared/services/blockchain-config-check.service.ts index 018c5e8e7c..9a163ee51c 100644 --- a/src/integration/blockchain/shared/services/blockchain-config-check.service.ts +++ b/src/integration/blockchain/shared/services/blockchain-config-check.service.ts @@ -20,7 +20,7 @@ export class BlockchainConfigCheckService { // reports what a client can actually tell us today: a missing Tatum API key (Cardano, Solana, Tron) and a // missing node URL (Bitcoin, Firo). Clients that build unconditionally report configured, so silence here // is not a full-coverage statement - @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.Worker, process: Process.BLOCKCHAIN_CONFIG_CHECK }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.WORKER, process: Process.BLOCKCHAIN_CONFIG_CHECK }) logUnconfiguredClients(): void { if (Config.environment !== Environment.PRD) return; diff --git a/src/integration/blockchain/spark/__tests__/spark-client.spec.ts b/src/integration/blockchain/spark/__tests__/spark-client.spec.ts index 5cef81dfef..ee3d218dfd 100644 --- a/src/integration/blockchain/spark/__tests__/spark-client.spec.ts +++ b/src/integration/blockchain/spark/__tests__/spark-client.spec.ts @@ -15,15 +15,15 @@ jest.mock('@buildonspark/spark-sdk', () => ({ SparkWallet: { initialize: jest.fn() }, })); -// Rolle des Prozesses, veränderbar je Test. Der Name muss mit `mock` beginnen, -// sonst verbietet Jest den Zugriff aus der gehobenen Modul-Fabrik heraus. +// The process role, set per test. The name has to start with `mock`, otherwise Jest forbids +// reaching it from the hoisted module factory. let mockCronRole = 'worker'; jest.mock('src/config/config', () => ({ - // Das Modul wird vollständig ersetzt, damit der Test nicht die ganze - // Konfigurationskette lädt. CronRole muss deshalb hier mitkommen: der Client - // liest es, um den Optimierungs-Timer an die Rolle zu binden. - CronRole: { All: 'all', Api: 'api', Worker: 'worker' }, + // The module is replaced entirely so the test does not pull in the whole configuration + // chain. CronRole therefore has to come along: the client reads it to bind the optimization + // timer to the role. + CronRole: { ALL: 'all', API: 'api', WORKER: 'worker' }, GetConfig: () => ({ cronRole: mockCronRole, blockchain: { @@ -92,6 +92,19 @@ describe('SparkClient', () => { clearInterval(interval.mock.results[0].value as NodeJS.Timeout); }); + it('runs it in the single-process role', () => { + // The mode every environment without a separate worker runs in, and the one the claim + // "behaviour is unchanged" rests on. Only the API role may skip this timer. + mockCronRole = 'all'; + const interval = jest.spyOn(global, 'setInterval'); + + new SparkClient(); + + expect(interval).toHaveBeenCalledTimes(1); + + clearInterval(interval.mock.results[0].value as NodeJS.Timeout); + }); + it('does not run it in the API process', () => { // Both processes run the same image against the same seed: without this // check the maintenance would run twice, unsynchronised. diff --git a/src/integration/blockchain/spark/spark-client.ts b/src/integration/blockchain/spark/spark-client.ts index f99811acd9..7a759569fb 100644 --- a/src/integration/blockchain/spark/spark-client.ts +++ b/src/integration/blockchain/spark/spark-client.ts @@ -229,7 +229,7 @@ export class SparkClient extends BlockchainClient { // On-chain wallet maintenance is global work: it must run in exactly one process. This // timer predates the scheduler and bypasses it, so without the role check the API process // would drive optimizeTokenOutputs against the same seed as the worker. - if (GetConfig().cronRole === CronRole.Api) return; + if (GetConfig().cronRole === CronRole.API) return; if (this.tokenOptimizationInterval) clearInterval(this.tokenOptimizationInterval); diff --git a/src/integration/blockchain/zano/services/zano.service.ts b/src/integration/blockchain/zano/services/zano.service.ts index 508c34e59c..b935824eb2 100644 --- a/src/integration/blockchain/zano/services/zano.service.ts +++ b/src/integration/blockchain/zano/services/zano.service.ts @@ -40,7 +40,7 @@ export class ZanoService extends BlockchainService implements OnModuleInit { } // --- JOBS --- // - @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.ZANO_ASSET_WHITELIST }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.WORKER, process: Process.ZANO_ASSET_WHITELIST }) async setupAssetWhitelist(): Promise { if (await this.isHealthy()) { const zanoTokens = await this.assetService.getTokens(Blockchain.ZANO); diff --git a/src/integration/exchange/controllers/exchange.controller.ts b/src/integration/exchange/controllers/exchange.controller.ts index 69fc9e5ff7..bed51491b6 100644 --- a/src/integration/exchange/controllers/exchange.controller.ts +++ b/src/integration/exchange/controllers/exchange.controller.ts @@ -171,7 +171,10 @@ export class ExchangeController { } // --- JOBS --- // - @DfxCron(CronExpression.EVERY_30_SECONDS, { scope: CronScope.Both, timeout: 1800 }) + // Api, not Both: `trades` is filled by POST :exchange/trade and read by GET trade/:id, both + // request paths of this controller. Nothing outside a request touches it, so in a process + // without ingress the map stays empty and the job has nothing to do. + @DfxCron(CronExpression.EVERY_30_SECONDS, { scope: CronScope.API, timeout: 1800 }) async checkTrades() { const openTrades = Object.values(this.trades).filter(({ status }) => status === TradeStatus.OPEN); for (const trade of openTrades) { diff --git a/src/integration/exchange/services/exchange-tx.service.ts b/src/integration/exchange/services/exchange-tx.service.ts index 41152b50f0..0bbb03a0b4 100644 --- a/src/integration/exchange/services/exchange-tx.service.ts +++ b/src/integration/exchange/services/exchange-tx.service.ts @@ -97,7 +97,7 @@ export class ExchangeTxService implements OnModuleInit { //*** JOBS ***// @DfxCron(CronExpression.EVERY_5_MINUTES, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.EXCHANGE_TX_SYNC, timeout: 1800, }) diff --git a/src/shared/services/__tests__/cron-registration.guard.spec.ts b/src/shared/services/__tests__/cron-registration.guard.spec.ts index 0b91b6c86b..c881ec9bcc 100644 --- a/src/shared/services/__tests__/cron-registration.guard.spec.ts +++ b/src/shared/services/__tests__/cron-registration.guard.spec.ts @@ -6,8 +6,7 @@ const SRC = join(__dirname, '..', '..', '..'); /** * Periodic work registered outside DfxCronService is invisible to the scope mechanism: it runs * in every process, which for anything writing to the database or driving business forward means - * running twice without a shared lock. Two native @Cron decorators and one bare setInterval had - * grown that way before the mechanism existed, and nothing would have flagged the next one. + * running twice without a shared lock. * * The check is syntactic on purpose. It asks whether a pattern occurs, not whether the code * behind it is safe, so its exception list has a natural ceiling and every entry is a deliberate diff --git a/src/shared/services/__tests__/dfx-cron.service.spec.ts b/src/shared/services/__tests__/dfx-cron.service.spec.ts index ba5a532fee..e851f3bf99 100644 --- a/src/shared/services/__tests__/dfx-cron.service.spec.ts +++ b/src/shared/services/__tests__/dfx-cron.service.spec.ts @@ -49,13 +49,13 @@ describe('DfxCronService', () => { const configuredJobs = [ providerWithJob('workerJob', { expression: CronExpression.EVERY_MINUTE, - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.MONITOR_EVENT_LOOP, }), // A worker job without `process` — DISABLED_PROCESSES cannot stop this one, only the role can. - providerWithJob('workerJobWithoutProcess', { expression: CronExpression.EVERY_MINUTE, scope: CronScope.Worker }), - providerWithJob('apiJob', { expression: CronExpression.EVERY_MINUTE, scope: CronScope.Api }), - providerWithJob('bothJob', { expression: CronExpression.EVERY_MINUTE, scope: CronScope.Both }), + providerWithJob('workerJobWithoutProcess', { expression: CronExpression.EVERY_MINUTE, scope: CronScope.WORKER }), + providerWithJob('apiJob', { expression: CronExpression.EVERY_MINUTE, scope: CronScope.API }), + providerWithJob('bothJob', { expression: CronExpression.EVERY_MINUTE, scope: CronScope.BOTH }), ]; function registeredJobNames(scheduler: SchedulerRegistry): string[] { diff --git a/src/shared/services/dfx-cron.service.ts b/src/shared/services/dfx-cron.service.ts index 2a52ff6920..0a3fa9edf4 100644 --- a/src/shared/services/dfx-cron.service.ts +++ b/src/shared/services/dfx-cron.service.ts @@ -78,14 +78,14 @@ export class DfxCronService implements OnModuleInit { */ private runsInThisRole(scope: CronScope): boolean { switch (Config.cronRole) { - case CronRole.All: + case CronRole.ALL: return true; - case CronRole.Api: - return scope === CronScope.Api || scope === CronScope.Both; + case CronRole.API: + return scope === CronScope.API || scope === CronScope.BOTH; - case CronRole.Worker: - return scope === CronScope.Worker || scope === CronScope.Both; + case CronRole.WORKER: + return scope === CronScope.WORKER || scope === CronScope.BOTH; } } diff --git a/src/shared/services/process.service.ts b/src/shared/services/process.service.ts index eb13504657..bbbcbd78d8 100644 --- a/src/shared/services/process.service.ts +++ b/src/shared/services/process.service.ts @@ -184,7 +184,7 @@ export class ProcessService implements OnModuleInit { await this.resyncStaffKycClearance(); } - @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800, scope: CronScope.Both }) + @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800, scope: CronScope.BOTH }) async resyncDisabledProcesses(): Promise { const allDisabledProcesses = [ ...(await this.settingService.getDisabledProcesses()), @@ -194,13 +194,13 @@ export class ProcessService implements OnModuleInit { DisabledProcesses = this.listToMap(allDisabledProcesses); } - @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800, scope: CronScope.Both }) + @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800, scope: CronScope.BOTH }) async resyncDeniedJwtAddresses(): Promise { const list = await this.settingService.getDeniedJwtAddresses(); DeniedJwtAddresses = new Set(list.map((a) => a.toLowerCase())); } - @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800, scope: CronScope.Both }) + @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800, scope: CronScope.BOTH }) async resyncDeniedJwtAccounts(): Promise { const list = await this.settingService.getDeniedJwtAccounts(); DeniedJwtAccounts = new Set(list); diff --git a/src/shared/utils/cron.ts b/src/shared/utils/cron.ts index 8e610d2b99..5895b48127 100644 --- a/src/shared/utils/cron.ts +++ b/src/shared/utils/cron.ts @@ -13,12 +13,12 @@ import { CustomCronExpression } from './custom-cron-expression'; */ export enum CronScope { /** Worker process only. The normal case: anything writing to the database or driving business forward. */ - Worker = 'worker', + WORKER = 'worker', /** * API process only. Maintains or measures state read exclusively from a request path, or * drives work bound to the connections that process holds open. */ - Api = 'api', + API = 'api', /** * Every process. Maintains or measures process-local state that both sides read. * @@ -27,7 +27,7 @@ export enum CronScope { * database or driving business forward does not - cron locks are per-process and cannot * prevent duplicate execution across processes. */ - Both = 'both', + BOTH = 'both', } export interface DfxCronOptParams { diff --git a/src/subdomains/core/accounting/services/ledger-booking-job.service.ts b/src/subdomains/core/accounting/services/ledger-booking-job.service.ts index 671c393a89..6647612dee 100644 --- a/src/subdomains/core/accounting/services/ledger-booking-job.service.ts +++ b/src/subdomains/core/accounting/services/ledger-booking-job.service.ts @@ -52,7 +52,7 @@ export class LedgerBookingJobService { } @DfxCron(CronExpression.EVERY_MINUTE, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.LEDGER_BOOKING_BANK_TX, timeout: 1800, }) @@ -63,7 +63,7 @@ export class LedgerBookingJobService { // ExchangeTx + ExchangeTrade are ONE @DfxCron method → one flag (Minor R8-1): deposit/withdrawal then trade @DfxCron(CronExpression.EVERY_MINUTE, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.LEDGER_BOOKING_EXCHANGE_TX, timeout: 1800, }) @@ -73,7 +73,7 @@ export class LedgerBookingJobService { } @DfxCron(CronExpression.EVERY_MINUTE, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.LEDGER_BOOKING_CRYPTO_INPUT, timeout: 1800, }) @@ -83,7 +83,7 @@ export class LedgerBookingJobService { } @DfxCron(CronExpression.EVERY_MINUTE, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.LEDGER_BOOKING_PAYOUT, timeout: 1800, }) @@ -93,7 +93,7 @@ export class LedgerBookingJobService { } @DfxCron(CronExpression.EVERY_MINUTE, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.LEDGER_BOOKING_BUY_CRYPTO, timeout: 1800, }) @@ -103,7 +103,7 @@ export class LedgerBookingJobService { } @DfxCron(CronExpression.EVERY_MINUTE, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.LEDGER_BOOKING_BUY_FIAT, timeout: 1800, }) @@ -114,7 +114,7 @@ export class LedgerBookingJobService { // §4.8 — bridge-only (skips exchange/DfxDex movements booked by their authoritative consumers) @DfxCron(CronExpression.EVERY_MINUTE, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.LEDGER_BOOKING_LIQUIDITY_MANAGEMENT, timeout: 1800, }) @@ -125,7 +125,7 @@ export class LedgerBookingJobService { // §4.8a — DfxDex purchase/sell on-chain swaps (own flag, Hard Constraint #5) @DfxCron(CronExpression.EVERY_MINUTE, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.LEDGER_BOOKING_LIQUIDITY_ORDER, timeout: 1800, }) @@ -136,7 +136,7 @@ export class LedgerBookingJobService { // §4.9 — arbitrage swaps (own flag, Hard Constraint #5) @DfxCron(CronExpression.EVERY_MINUTE, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.LEDGER_BOOKING_TRADING_ORDER, timeout: 1800, }) @@ -150,7 +150,7 @@ export class LedgerBookingJobService { // ("CoA bootstrap missing"). bootstrap() is idempotent (findOrCreate, §3), so a recurring re-run is a no-op // once complete. Pre-cutover the cutover run owns the bootstrap → gate on isLedgerReady. @DfxCron(CronExpression.EVERY_5_MINUTES, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.LEDGER_COA_BOOTSTRAP, timeout: 1800, }) diff --git a/src/subdomains/core/accounting/services/ledger-cutover.service.ts b/src/subdomains/core/accounting/services/ledger-cutover.service.ts index 6662787a5e..84d3393756 100644 --- a/src/subdomains/core/accounting/services/ledger-cutover.service.ts +++ b/src/subdomains/core/accounting/services/ledger-cutover.service.ts @@ -103,7 +103,7 @@ export class LedgerCutoverService { * a crash never breaks the boot/cron run, leaves `ledgerCutoverLogId` unset → all consumers no-op (§4 gate). * The cron no-ops immediately once the flag is set, so it effectively runs once and is otherwise idle. */ - @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.Worker, process: Process.LEDGER_CUTOVER }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.WORKER, process: Process.LEDGER_CUTOVER }) async run(): Promise { // Two deliberate checks: master switch (hard off, no DB) vs. already-cut-over (setting). Do not merge them. if (!Config.ledger.enabled) return; diff --git a/src/subdomains/core/accounting/services/ledger-mark-to-market.service.ts b/src/subdomains/core/accounting/services/ledger-mark-to-market.service.ts index 37e2521d20..1333547384 100644 --- a/src/subdomains/core/accounting/services/ledger-mark-to-market.service.ts +++ b/src/subdomains/core/accounting/services/ledger-mark-to-market.service.ts @@ -45,7 +45,7 @@ export class LedgerMarkToMarketService { private readonly ledgerLegRepository: LedgerLegRepository, ) {} - @DfxCron(CronExpression.EVERY_DAY_AT_4AM, { scope: CronScope.Worker, process: Process.LEDGER_MARK_TO_MARKET }) + @DfxCron(CronExpression.EVERY_DAY_AT_4AM, { scope: CronScope.WORKER, process: Process.LEDGER_MARK_TO_MARKET }) async run(): Promise { if (!(await this.jobService.isLedgerReady())) return; // cutover-gate (Blocker R1-6) applies here too diff --git a/src/subdomains/core/accounting/services/ledger-reconciliation.service.ts b/src/subdomains/core/accounting/services/ledger-reconciliation.service.ts index 0af99790df..f78cb3ef33 100644 --- a/src/subdomains/core/accounting/services/ledger-reconciliation.service.ts +++ b/src/subdomains/core/accounting/services/ledger-reconciliation.service.ts @@ -97,7 +97,7 @@ export class LedgerReconciliationService { private readonly refRewardService: RefRewardService, ) {} - @DfxCron(CronExpression.EVERY_DAY_AT_5AM, { scope: CronScope.Worker, process: Process.LEDGER_RECONCILIATION }) + @DfxCron(CronExpression.EVERY_DAY_AT_5AM, { scope: CronScope.WORKER, process: Process.LEDGER_RECONCILIATION }) async run(): Promise { if (!(await this.jobService.isLedgerReady())) return; // cutover-gate (Blocker R1-6) diff --git a/src/subdomains/core/aml/services/sanction.service.ts b/src/subdomains/core/aml/services/sanction.service.ts index 835750654f..fbe4f21b7f 100644 --- a/src/subdomains/core/aml/services/sanction.service.ts +++ b/src/subdomains/core/aml/services/sanction.service.ts @@ -40,7 +40,7 @@ export class SanctionService { ) {} // --- JOBS --- // - @DfxCron(CronExpression.EVERY_WEEKEND, { scope: CronScope.Worker, process: Process.SANCTION_SYNC }) + @DfxCron(CronExpression.EVERY_WEEKEND, { scope: CronScope.WORKER, process: Process.SANCTION_SYNC }) async syncList() { const filePath = Config.environment === Environment.LOC ? this.fileName : `/home/${this.fileName}`; diff --git a/src/subdomains/core/buy-crypto/process/services/buy-crypto-job.service.ts b/src/subdomains/core/buy-crypto/process/services/buy-crypto-job.service.ts index d11dd29009..135229025e 100644 --- a/src/subdomains/core/buy-crypto/process/services/buy-crypto-job.service.ts +++ b/src/subdomains/core/buy-crypto/process/services/buy-crypto-job.service.ts @@ -20,7 +20,7 @@ export class BuyCryptoJobService { private readonly buyCryptoPreparationService: BuyCryptoPreparationService, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.BUY_CRYPTO, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.BUY_CRYPTO, timeout: 7200 }) async process() { await this.buyCryptoRegistrationService.registerCryptoPayIn(); await this.buyCryptoRegistrationService.syncReturnTxId(); @@ -38,7 +38,7 @@ export class BuyCryptoJobService { } @DfxCron(CronExpression.EVERY_HOUR, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.BUY_CRYPTO_AGGREGATION, timeout: 7200, }) diff --git a/src/subdomains/core/buy-crypto/routes/buy/buy.service.ts b/src/subdomains/core/buy-crypto/routes/buy/buy.service.ts index 26b467a4dc..c573db0713 100644 --- a/src/subdomains/core/buy-crypto/routes/buy/buy.service.ts +++ b/src/subdomains/core/buy-crypto/routes/buy/buy.service.ts @@ -66,12 +66,12 @@ export class BuyService { ) {} // --- VOLUMES --- // - @DfxCron(CronExpression.EVERY_YEAR, { scope: CronScope.Worker }) + @DfxCron(CronExpression.EVERY_YEAR, { scope: CronScope.WORKER }) async resetAnnualVolumes(): Promise { await this.buyRepo.update({ annualVolume: Not(0) }, { annualVolume: 0 }); } - @DfxCron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, { scope: CronScope.Worker }) + @DfxCron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, { scope: CronScope.WORKER }) async resetMonthlyVolumes(): Promise { await this.buyRepo.update({ monthlyVolume: Not(0) }, { monthlyVolume: 0 }); } diff --git a/src/subdomains/core/buy-crypto/routes/swap/swap.service.ts b/src/subdomains/core/buy-crypto/routes/swap/swap.service.ts index 9ff8f7b77f..ff53d49cc5 100644 --- a/src/subdomains/core/buy-crypto/routes/swap/swap.service.ts +++ b/src/subdomains/core/buy-crypto/routes/swap/swap.service.ts @@ -85,12 +85,12 @@ export class SwapService { } // --- VOLUMES --- // - @DfxCron(CronExpression.EVERY_YEAR, { scope: CronScope.Worker }) + @DfxCron(CronExpression.EVERY_YEAR, { scope: CronScope.WORKER }) async resetAnnualVolumes(): Promise { await this.swapRepo.update({ annualVolume: Not(0) }, { annualVolume: 0 }); } - @DfxCron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, { scope: CronScope.Worker }) + @DfxCron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, { scope: CronScope.WORKER }) async resetMonthlyVolumes(): Promise { await this.swapRepo.update({ monthlyVolume: Not(0) }, { monthlyVolume: 0 }); } diff --git a/src/subdomains/core/custody/services/custody-job.service.ts b/src/subdomains/core/custody/services/custody-job.service.ts index b858a4ff04..2e6024b7cb 100644 --- a/src/subdomains/core/custody/services/custody-job.service.ts +++ b/src/subdomains/core/custody/services/custody-job.service.ts @@ -38,14 +38,14 @@ export class CustodyJobService { private readonly custodyOrderService: CustodyOrderService, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.CUSTODY }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.CUSTODY }) async handleOrders() { await this.executeOrder(); await this.executeStep(); await this.checkStep(); } - @DfxCron(CronExpression.EVERY_DAY_AT_4AM, { scope: CronScope.Worker, process: Process.CUSTODY }) + @DfxCron(CronExpression.EVERY_DAY_AT_4AM, { scope: CronScope.WORKER, process: Process.CUSTODY }) async resetExpiredConfirmedOrders() { const expiryDate = Util.daysBefore(Config.txRequestWaitingExpiryDays); diff --git a/src/subdomains/core/faucet-request/services/faucet-request.service.ts b/src/subdomains/core/faucet-request/services/faucet-request.service.ts index b889faca64..953bff8ad2 100644 --- a/src/subdomains/core/faucet-request/services/faucet-request.service.ts +++ b/src/subdomains/core/faucet-request/services/faucet-request.service.ts @@ -36,7 +36,7 @@ export class FaucetRequestService { return [Environment.DEV, Environment.LOC].includes(Config.environment) ? Blockchain.SEPOLIA : Blockchain.ETHEREUM; } - @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.Worker, process: Process.CRYPTO_PAYOUT }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.WORKER, process: Process.CRYPTO_PAYOUT }) async checkFaucetRequests(): Promise { const pendingFaucets = await this.faucetRequestRepo.find({ where: { status: FaucetRequestStatus.IN_PROGRESS } }); for (const faucet of pendingFaucets) { diff --git a/src/subdomains/core/history/controllers/transaction.controller.ts b/src/subdomains/core/history/controllers/transaction.controller.ts index 5ccb96d8a8..6e38cbf18d 100644 --- a/src/subdomains/core/history/controllers/transaction.controller.ts +++ b/src/subdomains/core/history/controllers/transaction.controller.ts @@ -116,7 +116,9 @@ export class TransactionController { ) {} // --- JOBS --- // - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Both }) + // Api, not Both: `refundList` is filled and read by the refund request paths of this + // controller only, so it stays empty in a process without ingress. + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.API }) checkLists() { for (const [key, refundData] of this.refundList.entries()) { if (!this.isRefundDataValid(refundData)) this.refundList.delete(key); diff --git a/src/subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.ts b/src/subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.ts index 9b77ca8cf1..da87a948ce 100644 --- a/src/subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.ts +++ b/src/subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.ts @@ -78,7 +78,7 @@ export class LiquidityManagementPipelineService { //*** JOBS ***// @DfxCron(CronExpression.EVERY_10_SECONDS, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.LIQUIDITY_MANAGEMENT, timeout: 1800, }) diff --git a/src/subdomains/core/liquidity-management/services/liquidity-management-rule.service.ts b/src/subdomains/core/liquidity-management/services/liquidity-management-rule.service.ts index 93e1af5913..eb1fc045c4 100644 --- a/src/subdomains/core/liquidity-management/services/liquidity-management-rule.service.ts +++ b/src/subdomains/core/liquidity-management/services/liquidity-management-rule.service.ts @@ -106,7 +106,7 @@ export class LiquidityManagementRuleService { //*** JOBS ***// @DfxCron(CronExpression.EVERY_5_MINUTES, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.LIQUIDITY_MANAGEMENT, timeout: 1800, }) diff --git a/src/subdomains/core/liquidity-management/services/liquidity-management.service.ts b/src/subdomains/core/liquidity-management/services/liquidity-management.service.ts index 32afddae8e..545dd86ca0 100644 --- a/src/subdomains/core/liquidity-management/services/liquidity-management.service.ts +++ b/src/subdomains/core/liquidity-management/services/liquidity-management.service.ts @@ -38,7 +38,7 @@ export class LiquidityManagementService { //*** JOBS ***// @DfxCron(CronExpression.EVERY_MINUTE, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.LIQUIDITY_MANAGEMENT_CHECK_BALANCES, timeout: 1800, }) diff --git a/src/subdomains/core/monitoring/monitor-connection-pool.service.ts b/src/subdomains/core/monitoring/monitor-connection-pool.service.ts index 338474a892..b21d687156 100644 --- a/src/subdomains/core/monitoring/monitor-connection-pool.service.ts +++ b/src/subdomains/core/monitoring/monitor-connection-pool.service.ts @@ -19,7 +19,7 @@ export class MonitorConnectionPoolService { this.dbConnectionPool = dbDriver.master; } - @DfxCron(CronExpression.EVERY_SECOND, { scope: CronScope.Both, process: Process.MONITOR_CONNECTION_POOL }) + @DfxCron(CronExpression.EVERY_SECOND, { scope: CronScope.BOTH, process: Process.MONITOR_CONNECTION_POOL }) monitorConnectionPool() { const dbOptions = Config.database as PostgresConnectionOptions; const dbMaxPoolConnections = dbOptions.poolSize ?? 10; @@ -37,7 +37,7 @@ export class MonitorConnectionPoolService { } } - @DfxCron(CronExpression.EVERY_10_SECONDS, { scope: CronScope.Both, process: Process.MONITOR_CONNECTION_POOL }) + @DfxCron(CronExpression.EVERY_10_SECONDS, { scope: CronScope.BOTH, process: Process.MONITOR_CONNECTION_POOL }) monitorConnectionPoolStatic() { const total = this.dbConnectionPool.totalCount; const idle = this.dbConnectionPool.idleCount; diff --git a/src/subdomains/core/monitoring/monitor-event-loop.service.ts b/src/subdomains/core/monitoring/monitor-event-loop.service.ts index c674213139..1093b7480d 100644 --- a/src/subdomains/core/monitoring/monitor-event-loop.service.ts +++ b/src/subdomains/core/monitoring/monitor-event-loop.service.ts @@ -22,7 +22,7 @@ export class MonitorEventLoopService implements OnModuleDestroy { this.histogram.disable(); } - @DfxCron(CronExpression.EVERY_10_SECONDS, { scope: CronScope.Both, process: Process.MONITOR_EVENT_LOOP }) + @DfxCron(CronExpression.EVERY_10_SECONDS, { scope: CronScope.BOTH, process: Process.MONITOR_EVENT_LOOP }) monitorEventLoop(): void { const toMs = (ns: number) => Math.round(ns / 1e6); diff --git a/src/subdomains/core/monitoring/observers/aml.observer.ts b/src/subdomains/core/monitoring/observers/aml.observer.ts index 050c6f80e5..87209d5664 100644 --- a/src/subdomains/core/monitoring/observers/aml.observer.ts +++ b/src/subdomains/core/monitoring/observers/aml.observer.ts @@ -31,7 +31,7 @@ export class AmlObserver extends MetricObserver { super(monitoringService, 'payment', 'aml'); } - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.MONITORING, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.MONITORING, timeout: 1800 }) async fetch() { const data = await this.getAmlData(); diff --git a/src/subdomains/core/monitoring/observers/bank.observer.ts b/src/subdomains/core/monitoring/observers/bank.observer.ts index 342338fc84..67a8205fb6 100644 --- a/src/subdomains/core/monitoring/observers/bank.observer.ts +++ b/src/subdomains/core/monitoring/observers/bank.observer.ts @@ -38,7 +38,7 @@ export class BankObserver extends MetricObserver { super(monitoringService, 'bank', 'balance'); } - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.MONITORING, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.MONITORING, timeout: 1800 }) async fetch() { let data = []; diff --git a/src/subdomains/core/monitoring/observers/checkout.observer.ts b/src/subdomains/core/monitoring/observers/checkout.observer.ts index 3bba38385e..d8f3561451 100644 --- a/src/subdomains/core/monitoring/observers/checkout.observer.ts +++ b/src/subdomains/core/monitoring/observers/checkout.observer.ts @@ -30,7 +30,7 @@ export class CheckoutObserver extends MetricObserver { super(monitoringService, 'checkout', 'balance'); } - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.MONITORING, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.MONITORING, timeout: 1800 }) async fetch() { if (!this.checkoutService.isAvailable()) { if (!this.unavailableWarningLogged) { diff --git a/src/subdomains/core/monitoring/observers/exchange.observer.ts b/src/subdomains/core/monitoring/observers/exchange.observer.ts index 8d473fa1c7..0506fcfb0d 100644 --- a/src/subdomains/core/monitoring/observers/exchange.observer.ts +++ b/src/subdomains/core/monitoring/observers/exchange.observer.ts @@ -27,7 +27,7 @@ export class ExchangeObserver extends MetricObserver { super(monitoringService, 'exchange', 'volume'); } - @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.MONITORING, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.WORKER, process: Process.MONITORING, timeout: 1800 }) async fetch() { if (DisabledProcess(Process.MONITORING)) return; diff --git a/src/subdomains/core/monitoring/observers/external-services.observer.ts b/src/subdomains/core/monitoring/observers/external-services.observer.ts index 815b6440c3..0278e501f8 100644 --- a/src/subdomains/core/monitoring/observers/external-services.observer.ts +++ b/src/subdomains/core/monitoring/observers/external-services.observer.ts @@ -31,7 +31,7 @@ export class ExternalServicesObserver extends MetricObserver { super(monitoringService, 'liquidity', 'trading'); } - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.MONITORING, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.MONITORING, timeout: 1800 }) async fetch() { const data = await this.getLiquidityData(); diff --git a/src/subdomains/core/monitoring/observers/node-balance.observer.ts b/src/subdomains/core/monitoring/observers/node-balance.observer.ts index b66f03c017..6d9e7e10ec 100644 --- a/src/subdomains/core/monitoring/observers/node-balance.observer.ts +++ b/src/subdomains/core/monitoring/observers/node-balance.observer.ts @@ -31,7 +31,7 @@ export class NodeBalanceObserver extends MetricObserver { this.bitcoinClient = bitcoinService.getDefaultClient(BitcoinNodeType.BTC_INPUT); } - @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.MONITORING, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.WORKER, process: Process.MONITORING, timeout: 1800 }) async fetch(): Promise { const data = await this.getNode(); diff --git a/src/subdomains/core/monitoring/observers/node-health.observer.ts b/src/subdomains/core/monitoring/observers/node-health.observer.ts index f6462ea8c8..f7aa07c8b3 100644 --- a/src/subdomains/core/monitoring/observers/node-health.observer.ts +++ b/src/subdomains/core/monitoring/observers/node-health.observer.ts @@ -45,7 +45,7 @@ export class NodeHealthObserver extends MetricObserver { this.emit(data); } - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.MONITORING, timeout: 360 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.MONITORING, timeout: 360 }) async fetch(): Promise { const previousState = this.data; diff --git a/src/subdomains/core/monitoring/observers/payment.observer.ts b/src/subdomains/core/monitoring/observers/payment.observer.ts index ecbefe690b..b3bc83a1a0 100644 --- a/src/subdomains/core/monitoring/observers/payment.observer.ts +++ b/src/subdomains/core/monitoring/observers/payment.observer.ts @@ -51,7 +51,7 @@ export class PaymentObserver extends MetricObserver { super(monitoringService, 'payment', 'combined'); } - @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.MONITORING, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.WORKER, process: Process.MONITORING, timeout: 1800 }) async fetch() { const data = await this.getPayment(); diff --git a/src/subdomains/core/monitoring/observers/realunit-w2w-gas.observer.ts b/src/subdomains/core/monitoring/observers/realunit-w2w-gas.observer.ts index b14f1167d0..5685ab4a18 100644 --- a/src/subdomains/core/monitoring/observers/realunit-w2w-gas.observer.ts +++ b/src/subdomains/core/monitoring/observers/realunit-w2w-gas.observer.ts @@ -40,7 +40,7 @@ export class RealUnitW2wGasObserver extends MetricObserver { super(monitoringService, 'realUnit', 'w2wGasBalance'); } - @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.MONITORING, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.WORKER, process: Process.MONITORING, timeout: 1800 }) async fetch(): Promise { const data = await this.getData(); diff --git a/src/subdomains/core/monitoring/observers/user.observer.ts b/src/subdomains/core/monitoring/observers/user.observer.ts index 1a98cd006d..62cbfa3300 100644 --- a/src/subdomains/core/monitoring/observers/user.observer.ts +++ b/src/subdomains/core/monitoring/observers/user.observer.ts @@ -27,7 +27,7 @@ export class UserObserver extends MetricObserver { super(monitoringService, 'user', 'kyc'); } - @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.MONITORING, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.WORKER, process: Process.MONITORING, timeout: 1800 }) async fetch(): Promise { const data = await this.getUser(); diff --git a/src/subdomains/core/payment-link/services/payment-cron.service.ts b/src/subdomains/core/payment-link/services/payment-cron.service.ts index f77886891f..e134185417 100644 --- a/src/subdomains/core/payment-link/services/payment-cron.service.ts +++ b/src/subdomains/core/payment-link/services/payment-cron.service.ts @@ -19,11 +19,11 @@ export class PaymentCronService { // Api, not Worker: this job and checkTxConfirmations both end up in // PaymentLinkPaymentService.doSave(), which resolves the AsyncMap that PaymentLinkController's // waitForPayment is waiting on and pushes the device activation into the RxJS subject the - // gateway delivers to its connected clients. Both are confined to the process holding those - // connections, and the worker holds none. `Both` is no option either: the jobs write to the - // database and trigger merchant webhooks, which two processes without a shared lock would do - // twice. - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Api, process: Process.PAYMENT_EXPIRATION }) + // gateway delivers to its connected clients. Both mechanisms only reach clients connected to + // the very process the job runs in, which is what CronScope.API expresses. `Both` is no option + // either: the jobs write to the database and trigger merchant webhooks, which two processes + // without a shared lock would do twice. + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.API, process: Process.PAYMENT_EXPIRATION }) async processExpiredPayments(): Promise { await this.paymentLinkPaymentService.processExpiredPayments(); await this.paymentActivationService.processExpiredActivations(); @@ -31,12 +31,12 @@ export class PaymentCronService { } // Api for the same reason as processExpiredPayments above. - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Api, process: Process.PAYMENT_CONFIRMATIONS }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.API, process: Process.PAYMENT_CONFIRMATIONS }) async checkTxConfirmations(): Promise { await this.paymentLinkPaymentService.checkTxConfirmations(); } - @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, process: Process.PAYMENT_FORWARDING }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.WORKER, process: Process.PAYMENT_FORWARDING }) async forwardDeposits(): Promise { await this.paymentBalanceService.forwardDeposits(); } diff --git a/src/subdomains/core/payment-link/services/payment-link-fee.service.ts b/src/subdomains/core/payment-link/services/payment-link-fee.service.ts index bab5ceee52..f2e495331a 100644 --- a/src/subdomains/core/payment-link/services/payment-link-fee.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-fee.service.ts @@ -38,7 +38,7 @@ export class PaymentLinkFeeService implements OnModuleInit { } // --- JOBS --- // - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Both, process: Process.UPDATE_BLOCKCHAIN_FEE }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.BOTH, process: Process.UPDATE_BLOCKCHAIN_FEE }) async updateFees(): Promise { if (GetConfig().environment === Environment.LOC) return; diff --git a/src/subdomains/core/referral/process/ref.service.ts b/src/subdomains/core/referral/process/ref.service.ts index 3af692a81d..61aeea4a76 100644 --- a/src/subdomains/core/referral/process/ref.service.ts +++ b/src/subdomains/core/referral/process/ref.service.ts @@ -16,7 +16,7 @@ export class RefService { constructor(private readonly repo: RefRepository) {} - @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, process: Process.REF_CLEANUP, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.WORKER, process: Process.REF_CLEANUP, timeout: 7200 }) async checkRefs(): Promise { const expirationDate = Util.daysBefore(this.refExpirationDays); diff --git a/src/subdomains/core/referral/reward/services/ref-reward-job.service.ts b/src/subdomains/core/referral/reward/services/ref-reward-job.service.ts index 6863b0972f..033b95ff80 100644 --- a/src/subdomains/core/referral/reward/services/ref-reward-job.service.ts +++ b/src/subdomains/core/referral/reward/services/ref-reward-job.service.ts @@ -16,12 +16,12 @@ export class RefRewardJobService { private readonly refRewardService: RefRewardService, ) {} - @DfxCron(CronExpression.EVERY_DAY_AT_6AM, { scope: CronScope.Worker, process: Process.REF_PAYOUT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_DAY_AT_6AM, { scope: CronScope.WORKER, process: Process.REF_PAYOUT, timeout: 1800 }) async createPendingRefRewards() { await this.refRewardService.createPendingRefRewards(); } - @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.REF_PAYOUT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.WORKER, process: Process.REF_PAYOUT, timeout: 1800 }) async processPendingRefRewards() { await this.refRewardDexService.secureLiquidity(); await this.refRewardOutService.checkPaidTransaction(); diff --git a/src/subdomains/core/sell-crypto/process/services/buy-fiat-job.service.ts b/src/subdomains/core/sell-crypto/process/services/buy-fiat-job.service.ts index 55ef430b2c..175725084b 100644 --- a/src/subdomains/core/sell-crypto/process/services/buy-fiat-job.service.ts +++ b/src/subdomains/core/sell-crypto/process/services/buy-fiat-job.service.ts @@ -12,7 +12,7 @@ export class BuyFiatJobService { private readonly buyFiatPreparationService: BuyFiatPreparationService, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.BUY_FIAT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.BUY_FIAT, timeout: 1800 }) async checkCryptoPayIn() { await this.buyFiatRegistrationService.registerSellPayIn(); await this.buyFiatRegistrationService.syncReturnTxId(); @@ -26,7 +26,7 @@ export class BuyFiatJobService { await this.buyFiatPreparationService.chargebackTx(); } - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.BUY_FIAT, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.BUY_FIAT, timeout: 7200 }) async addFiatOutputs(): Promise { await this.buyFiatPreparationService.addFiatOutputs(); } diff --git a/src/subdomains/core/sell-crypto/process/services/buy-fiat-notification.service.ts b/src/subdomains/core/sell-crypto/process/services/buy-fiat-notification.service.ts index 7835b4ecfa..add8170b9b 100644 --- a/src/subdomains/core/sell-crypto/process/services/buy-fiat-notification.service.ts +++ b/src/subdomains/core/sell-crypto/process/services/buy-fiat-notification.service.ts @@ -30,7 +30,7 @@ export class BuyFiatNotificationService { private readonly notificationService: NotificationService, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.BUY_FIAT_MAIL, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.BUY_FIAT_MAIL, timeout: 1800 }) async sendNotificationMails(): Promise { await this.paymentCompleted(); await this.chargebackInitiated(); diff --git a/src/subdomains/core/sell-crypto/route/sell.service.ts b/src/subdomains/core/sell-crypto/route/sell.service.ts index 3b52153664..a389491662 100644 --- a/src/subdomains/core/sell-crypto/route/sell.service.ts +++ b/src/subdomains/core/sell-crypto/route/sell.service.ts @@ -223,12 +223,12 @@ export class SellService { } // --- VOLUMES --- // - @DfxCron(CronExpression.EVERY_YEAR, { scope: CronScope.Worker }) + @DfxCron(CronExpression.EVERY_YEAR, { scope: CronScope.WORKER }) async resetAnnualVolumes(): Promise { await this.sellRepo.update({ annualVolume: Not(0) }, { annualVolume: 0 }); } - @DfxCron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, { scope: CronScope.Worker }) + @DfxCron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, { scope: CronScope.WORKER }) async resetMonthlyVolumes(): Promise { await this.sellRepo.update({ monthlyVolume: Not(0) }, { monthlyVolume: 0 }); } diff --git a/src/subdomains/core/statistic/statistic.service.ts b/src/subdomains/core/statistic/statistic.service.ts index 52adae78a0..d025b9014f 100644 --- a/src/subdomains/core/statistic/statistic.service.ts +++ b/src/subdomains/core/statistic/statistic.service.ts @@ -25,7 +25,7 @@ export class StatisticService implements OnModuleInit { void this.doUpdate(); } - @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Api, process: Process.UPDATE_STATISTIC, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.API, process: Process.UPDATE_STATISTIC, timeout: 7200 }) async doUpdate(): Promise { this.statistic = { totalVolume: { diff --git a/src/subdomains/core/trading/services/trading-job.service.ts b/src/subdomains/core/trading/services/trading-job.service.ts index 200e706706..4e895fb7a8 100644 --- a/src/subdomains/core/trading/services/trading-job.service.ts +++ b/src/subdomains/core/trading/services/trading-job.service.ts @@ -14,19 +14,19 @@ export class TradingJobService { // --- RULES --- // - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.TRADING, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.TRADING, timeout: 1800 }) async processRules() { await this.ruleService.processRules(); } - @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.Worker, process: Process.TRADING, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.WORKER, process: Process.TRADING, timeout: 1800 }) async reactivateRules(): Promise { await this.ruleService.reactivateRules(); } // --- ORDERS --- // - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.TRADING, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.TRADING, timeout: 1800 }) async processOrders() { await this.orderService.processOrders(); } diff --git a/src/subdomains/generic/admin/admin.service.ts b/src/subdomains/generic/admin/admin.service.ts index 2182b1b516..c74b111635 100644 --- a/src/subdomains/generic/admin/admin.service.ts +++ b/src/subdomains/generic/admin/admin.service.ts @@ -79,7 +79,7 @@ export class AdminService { } } - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.PAY_OUT, timeout: 3600 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.PAY_OUT, timeout: 3600 }) async completeLiquidityOrders() { for (const context of Object.values(PayoutRequestContext)) { const lContext = context as unknown as LiquidityOrderContext; diff --git a/src/subdomains/generic/kyc/services/kyc-notification.service.ts b/src/subdomains/generic/kyc/services/kyc-notification.service.ts index 9f14ea617c..014481e755 100644 --- a/src/subdomains/generic/kyc/services/kyc-notification.service.ts +++ b/src/subdomains/generic/kyc/services/kyc-notification.service.ts @@ -26,7 +26,7 @@ export class KycNotificationService { private readonly webhookService: WebhookService, ) {} - @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, process: Process.KYC_MAIL, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.WORKER, process: Process.KYC_MAIL, timeout: 1800 }) async sendNotificationMails(): Promise { await this.autoKycStepReminder(); } diff --git a/src/subdomains/generic/kyc/services/kyc.service.ts b/src/subdomains/generic/kyc/services/kyc.service.ts index 8e742fc32b..585bc094ed 100644 --- a/src/subdomains/generic/kyc/services/kyc.service.ts +++ b/src/subdomains/generic/kyc/services/kyc.service.ts @@ -134,7 +134,7 @@ export class KycService { this.webhookQueue = new QueueHandler(); } - @DfxCron(CronExpression.EVERY_DAY_AT_4AM, { scope: CronScope.Worker, process: Process.KYC }) + @DfxCron(CronExpression.EVERY_DAY_AT_4AM, { scope: CronScope.WORKER, process: Process.KYC }) async checkIdentSteps(): Promise { const expiredIdentSteps = await this.kycStepRepo.find({ where: { @@ -162,7 +162,7 @@ export class KycService { } } - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.KYC }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.KYC }) async reviewKycSteps(): Promise { await this.reviewNationalityStep(); await this.reviewIdentSteps(); diff --git a/src/subdomains/generic/kyc/services/tfa.service.ts b/src/subdomains/generic/kyc/services/tfa.service.ts index 9b4af160a5..c1ae401bbf 100644 --- a/src/subdomains/generic/kyc/services/tfa.service.ts +++ b/src/subdomains/generic/kyc/services/tfa.service.ts @@ -50,7 +50,7 @@ export class TfaService { private readonly notificationService: NotificationService, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Both, process: Process.TFA_CACHE }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.BOTH, process: Process.TFA_CACHE }) processCleanupSecretCache() { const now = new Date(); diff --git a/src/subdomains/generic/user/models/auth/auth-lnurl.service.ts b/src/subdomains/generic/user/models/auth/auth-lnurl.service.ts index f4a486bc22..05ac59373e 100644 --- a/src/subdomains/generic/user/models/auth/auth-lnurl.service.ts +++ b/src/subdomains/generic/user/models/auth/auth-lnurl.service.ts @@ -36,7 +36,7 @@ export class AuthLnUrlService { private readonly ipLogService: IpLogService, ) {} - @DfxCron(CronExpression.EVERY_30_SECONDS, { scope: CronScope.Both, process: Process.LNURL_AUTH_CACHE }) + @DfxCron(CronExpression.EVERY_30_SECONDS, { scope: CronScope.BOTH, process: Process.LNURL_AUTH_CACHE }) processCleanupAccessToken() { const before30SecTime = Util.secondsBefore(30).getTime(); @@ -47,7 +47,7 @@ export class AuthLnUrlService { keysToBeDeleted.forEach((k) => this.authCache.delete(k)); } - @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.Both, process: Process.LNURL_AUTH_CACHE }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.BOTH, process: Process.LNURL_AUTH_CACHE }) processCleanupAuthCache() { const before5MinTime = Util.minutesBefore(5).getTime(); diff --git a/src/subdomains/generic/user/models/auth/auth.service.ts b/src/subdomains/generic/user/models/auth/auth.service.ts index f77c22e9d8..2dba8df2d2 100644 --- a/src/subdomains/generic/user/models/auth/auth.service.ts +++ b/src/subdomains/generic/user/models/auth/auth.service.ts @@ -98,7 +98,7 @@ export class AuthService { @Inject(forwardRef(() => KycService)) private readonly kycService: KycService, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Both }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.BOTH }) checkLists() { for (const [key, challenge] of this.challengeList.entries()) { if (!this.isChallengeValid(challenge)) { diff --git a/src/subdomains/generic/user/models/bank-data/bank-data.service.ts b/src/subdomains/generic/user/models/bank-data/bank-data.service.ts index cb5f273823..af1714620b 100644 --- a/src/subdomains/generic/user/models/bank-data/bank-data.service.ts +++ b/src/subdomains/generic/user/models/bank-data/bank-data.service.ts @@ -47,7 +47,7 @@ export class BankDataService { ) {} @DfxCron(CronExpression.EVERY_MINUTE, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.BANK_DATA_VERIFICATION, timeout: 1800, }) diff --git a/src/subdomains/generic/user/models/organization/organization.service.ts b/src/subdomains/generic/user/models/organization/organization.service.ts index e0196eff70..0227d6fd0c 100644 --- a/src/subdomains/generic/user/models/organization/organization.service.ts +++ b/src/subdomains/generic/user/models/organization/organization.service.ts @@ -21,7 +21,7 @@ export class OrganizationService { private readonly userDataRepo: UserDataRepository, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.ORGANIZATION_SYNC, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.ORGANIZATION_SYNC, timeout: 1800 }) async syncOrganization() { const entities = await this.userDataRepo.findBy({ organization: { id: IsNull() }, diff --git a/src/subdomains/generic/user/models/user-data/jwt-revocation-sync.service.ts b/src/subdomains/generic/user/models/user-data/jwt-revocation-sync.service.ts index 21813a0f15..a333eb5e3d 100644 --- a/src/subdomains/generic/user/models/user-data/jwt-revocation-sync.service.ts +++ b/src/subdomains/generic/user/models/user-data/jwt-revocation-sync.service.ts @@ -27,7 +27,7 @@ export class JwtRevocationSyncService { // Runs every minute: fast revocation of a blocked or compromised account is a security requirement that // warrants the security-revocation exception to the "prefer 15min" cron guideline. @DfxCron(CronExpression.EVERY_MINUTE, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.JWT_REVOCATION_SYNC, timeout: 1800, }) diff --git a/src/subdomains/generic/user/models/user-data/user-data-job.service.ts b/src/subdomains/generic/user/models/user-data/user-data-job.service.ts index debd2014a9..5559d9c69f 100644 --- a/src/subdomains/generic/user/models/user-data/user-data-job.service.ts +++ b/src/subdomains/generic/user/models/user-data/user-data-job.service.ts @@ -15,7 +15,7 @@ import { UserDataRepository } from './user-data.repository'; export class UserDataJobService { constructor(private readonly userDataRepo: UserDataRepository) {} - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.USER_DATA, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.USER_DATA, timeout: 1800 }) async fillUserData() { await this.bankTxVerification(); await this.setAccountOpener(); diff --git a/src/subdomains/generic/user/models/user-data/user-data-notification.service.ts b/src/subdomains/generic/user/models/user-data/user-data-notification.service.ts index e403f6063c..017b5ce23c 100644 --- a/src/subdomains/generic/user/models/user-data/user-data-notification.service.ts +++ b/src/subdomains/generic/user/models/user-data/user-data-notification.service.ts @@ -20,7 +20,7 @@ export class UserDataNotificationService { private readonly notificationService: NotificationService, ) {} - @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, process: Process.BLACK_SQUAD_MAIL, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.WORKER, process: Process.BLACK_SQUAD_MAIL, timeout: 1800 }) async sendNotificationMails(): Promise { await this.blackSquadInvitation(); } diff --git a/src/subdomains/generic/user/models/user-data/user-data.service.ts b/src/subdomains/generic/user/models/user-data/user-data.service.ts index 20f459a7fe..32523dcbd1 100644 --- a/src/subdomains/generic/user/models/user-data/user-data.service.ts +++ b/src/subdomains/generic/user/models/user-data/user-data.service.ts @@ -833,7 +833,7 @@ export class UserDataService { return this.doUpdateUserMail(userData, cacheEntry.mail); } - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Both }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.BOTH }) processCleanupMailSecretCache(): void { const now = new Date(); @@ -1191,7 +1191,7 @@ export class UserDataService { } // --- VOLUMES --- // - @DfxCron(CronExpression.EVERY_YEAR, { scope: CronScope.Worker }) + @DfxCron(CronExpression.EVERY_YEAR, { scope: CronScope.WORKER }) async resetAnnualVolumes(): Promise { await this.userDataRepo.update( [{ annualBuyVolume: Not(0) }, { annualSellVolume: Not(0) }, { annualCryptoVolume: Not(0) }], @@ -1199,7 +1199,7 @@ export class UserDataService { ); } - @DfxCron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, { scope: CronScope.Worker }) + @DfxCron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, { scope: CronScope.WORKER }) async resetMonthlyVolumes(): Promise { await this.userDataRepo.update( [{ monthlyBuyVolume: Not(0) }, { monthlySellVolume: Not(0) }, { monthlyCryptoVolume: Not(0) }], diff --git a/src/subdomains/generic/user/models/user/user-job.service.ts b/src/subdomains/generic/user/models/user/user-job.service.ts index abd2112b9b..ba82bed7d0 100644 --- a/src/subdomains/generic/user/models/user/user-job.service.ts +++ b/src/subdomains/generic/user/models/user/user-job.service.ts @@ -10,7 +10,7 @@ import { UserRepository } from './user.repository'; export class UserJobService { constructor(private readonly userRepo: UserRepository) {} - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.USER, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.USER, timeout: 1800 }) async fillUser() { await this.approveUser(); } diff --git a/src/subdomains/generic/user/models/user/user.service.ts b/src/subdomains/generic/user/models/user/user.service.ts index d4a5ddd17d..d6bba889a4 100644 --- a/src/subdomains/generic/user/models/user/user.service.ts +++ b/src/subdomains/generic/user/models/user/user.service.ts @@ -541,7 +541,7 @@ export class UserService { } // --- VOLUMES --- // - @DfxCron(CronExpression.EVERY_YEAR, { scope: CronScope.Worker }) + @DfxCron(CronExpression.EVERY_YEAR, { scope: CronScope.WORKER }) async resetAnnualVolumes(): Promise { await this.userRepo.update( [{ annualBuyVolume: Not(0) }, { annualSellVolume: Not(0) }, { annualCryptoVolume: Not(0) }], @@ -549,7 +549,7 @@ export class UserService { ); } - @DfxCron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, { scope: CronScope.Worker }) + @DfxCron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT, { scope: CronScope.WORKER }) async resetMonthlyVolumes(): Promise { await this.userRepo.update( [{ monthlyBuyVolume: Not(0) }, { monthlySellVolume: Not(0) }, { monthlyCryptoVolume: Not(0) }], diff --git a/src/subdomains/generic/user/services/webhook/webhook-notification.service.ts b/src/subdomains/generic/user/services/webhook/webhook-notification.service.ts index 7eb14d7b9f..01981a4911 100644 --- a/src/subdomains/generic/user/services/webhook/webhook-notification.service.ts +++ b/src/subdomains/generic/user/services/webhook/webhook-notification.service.ts @@ -23,7 +23,7 @@ export class WebhookNotificationService { private readonly notificationService: NotificationService, ) {} - @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.Worker, process: Process.WEBHOOK, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.WORKER, process: Process.WEBHOOK, timeout: 1800 }) async sendWebhooks() { await this.sendOpenWebhooks(); } diff --git a/src/subdomains/supporting/bank-tx/bank-tx-return/bank-tx-return-notification.service.ts b/src/subdomains/supporting/bank-tx/bank-tx-return/bank-tx-return-notification.service.ts index a27f851d94..238f30e8e5 100644 --- a/src/subdomains/supporting/bank-tx/bank-tx-return/bank-tx-return-notification.service.ts +++ b/src/subdomains/supporting/bank-tx/bank-tx-return/bank-tx-return-notification.service.ts @@ -19,7 +19,7 @@ export class BankTxReturnNotificationService { ) {} @DfxCron(CronExpression.EVERY_MINUTE, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.BANK_TX_RETURN_MAIL, timeout: 1800, }) diff --git a/src/subdomains/supporting/bank-tx/bank-tx-return/bank-tx-return.service.ts b/src/subdomains/supporting/bank-tx/bank-tx-return/bank-tx-return.service.ts index f14ef8b59d..413785001e 100644 --- a/src/subdomains/supporting/bank-tx/bank-tx-return/bank-tx-return.service.ts +++ b/src/subdomains/supporting/bank-tx/bank-tx-return/bank-tx-return.service.ts @@ -35,7 +35,7 @@ export class BankTxReturnService { private readonly fiatService: FiatService, ) {} - @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.Worker, process: Process.BANK_TX_RETURN, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.WORKER, process: Process.BANK_TX_RETURN, timeout: 1800 }) async fillBankTxReturn() { await this.chargebackTx(); await this.setFiatAmounts(); diff --git a/src/subdomains/supporting/bank-tx/bank-tx/services/bank-tx.service.ts b/src/subdomains/supporting/bank-tx/bank-tx/services/bank-tx.service.ts index 6f67bb8871..1e05d35ed9 100644 --- a/src/subdomains/supporting/bank-tx/bank-tx/services/bank-tx.service.ts +++ b/src/subdomains/supporting/bank-tx/bank-tx/services/bank-tx.service.ts @@ -132,7 +132,7 @@ export class BankTxService implements OnModuleInit { } // --- TRANSACTION HANDLING --- // - @DfxCron(CronExpression.EVERY_30_SECONDS, { scope: CronScope.Worker, timeout: 3600, process: Process.BANK_TX }) + @DfxCron(CronExpression.EVERY_30_SECONDS, { scope: CronScope.WORKER, timeout: 3600, process: Process.BANK_TX }) async checkBankTx(): Promise { try { await this.checkTransactions(); @@ -148,7 +148,7 @@ export class BankTxService implements OnModuleInit { await this.fillBankTx(); } - @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.Worker, process: Process.BANK_TX }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.WORKER, process: Process.BANK_TX }) async enrichYapealTransactions(): Promise { const transactions = await this.bankTxRepo.find({ where: { familyCode: 'CCRD' }, // credit card => wrong data diff --git a/src/subdomains/supporting/bank/bank-account/bank-account.service.ts b/src/subdomains/supporting/bank/bank-account/bank-account.service.ts index c994e58ee4..04b3487bbf 100644 --- a/src/subdomains/supporting/bank/bank-account/bank-account.service.ts +++ b/src/subdomains/supporting/bank/bank-account/bank-account.service.ts @@ -35,7 +35,7 @@ export class BankAccountService { // --- INTERNAL METHODS --- // - @DfxCron(CronExpression.EVERY_WEEK, { scope: CronScope.Worker, process: Process.BANK_ACCOUNT, timeout: 3600 }) + @DfxCron(CronExpression.EVERY_WEEK, { scope: CronScope.WORKER, process: Process.BANK_ACCOUNT, timeout: 3600 }) async checkFailedBankAccounts(): Promise { const failedBankAccounts = await this.bankAccountRepo.findBy({ returnCode: 256 }); for (const bankAccount of failedBankAccounts) { @@ -43,7 +43,7 @@ export class BankAccountService { } } - @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, process: Process.BANK_ACCOUNT, timeout: 3600 }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.WORKER, process: Process.BANK_ACCOUNT, timeout: 3600 }) async reloadErrorBankAccounts(): Promise { const bankAccounts = await this.bankAccountRepo.findBy({ result: Like('Error:%') }); for (const bankAccount of bankAccounts) { @@ -51,7 +51,7 @@ export class BankAccountService { } } - @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.BANK_ACCOUNT, timeout: 3600 }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.WORKER, process: Process.BANK_ACCOUNT, timeout: 3600 }) async reloadUncheckedBankAccounts(): Promise { const bankAccounts = await this.bankAccountRepo.findBy({ result: IsNull(), iban: Not(IsNull()) }); for (const bankAccount of bankAccounts) { diff --git a/src/subdomains/supporting/bank/virtual-iban/virtual-iban-frick-issuance-reconciliation.service.ts b/src/subdomains/supporting/bank/virtual-iban/virtual-iban-frick-issuance-reconciliation.service.ts index 72f3eb44ef..4a3bb2e8e3 100644 --- a/src/subdomains/supporting/bank/virtual-iban/virtual-iban-frick-issuance-reconciliation.service.ts +++ b/src/subdomains/supporting/bank/virtual-iban/virtual-iban-frick-issuance-reconciliation.service.ts @@ -88,7 +88,7 @@ export class VirtualIbanFrickIssuanceReconciliationService { * and external cleanup targets exact vIBAN identities, making repeated work fail closed or idempotent. */ @DfxCron(CronExpression.EVERY_HOUR, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.VIRTUAL_IBAN_FRICK_ISSUANCE_RECONCILIATION, timeout: 1800, }) diff --git a/src/subdomains/supporting/dashboard/dashboard-financial.service.ts b/src/subdomains/supporting/dashboard/dashboard-financial.service.ts index 69d1a42e0c..bc7a48572c 100644 --- a/src/subdomains/supporting/dashboard/dashboard-financial.service.ts +++ b/src/subdomains/supporting/dashboard/dashboard-financial.service.ts @@ -134,7 +134,7 @@ export class DashboardFinancialService { /** * Fills LatestBalanceStore from the most recent FinancialDataLog entry, so that * GET /v1/dashboard/financial/latest keeps answering from process memory without touching the - * database - the property that turned a 23 ms median with a 1'989 ms p95 into a field read. + * database - see getLatestBalance below, which reads the store and nothing else. * * Scope Api, because the store it fills is process-local and its only reader is that endpoint. * The expensive part stays where it was: the financial aggregation writing the log entry runs in @@ -145,8 +145,13 @@ export class DashboardFinancialService { * This is not a fallback taken only when the store is empty: such a path would run once per * deployment and would never be exercised. It runs every minute, in normal operation as much as * after a failure. + * + * The trade-off is one extra read per minute compared to the write-through this replaces, which + * was handed its inputs by the caller. It applies in the single-process role too, where both + * jobs run in the same process - accepted deliberately: sharing state between the two would tie + * the endpoint's cache back to the aggregation it was decoupled from. */ - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Api, process: Process.LATEST_BALANCE_CACHE }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.API, process: Process.LATEST_BALANCE_CACHE }) async refreshLatestBalance(): Promise { const latest = await this.logService.getLatestFinancialLog(); if (!latest) return; diff --git a/src/subdomains/supporting/dashboard/latest-balance.store.ts b/src/subdomains/supporting/dashboard/latest-balance.store.ts index c97e6c7249..964918d1cb 100644 --- a/src/subdomains/supporting/dashboard/latest-balance.store.ts +++ b/src/subdomains/supporting/dashboard/latest-balance.store.ts @@ -7,7 +7,7 @@ import { LatestBalanceResponseDto } from './dto/financial-log.dto'; * by GET /v1/dashboard/financial/latest. Exactly one entry, replaced wholesale on every job run: * no TTL, no eviction, no size cap. * - * The store is process-local, and so is the job filling it: it carries CronScope.Api, so it runs + * The store is process-local, and so is the job filling it: it carries CronScope.API, so it runs * in whichever process serves the requests reading it. There is one writer per process and no * cross-process state to reconcile - both derive the same value from the same row. * diff --git a/src/subdomains/supporting/dex/services/dex.service.ts b/src/subdomains/supporting/dex/services/dex.service.ts index 1b18435239..15902703f8 100644 --- a/src/subdomains/supporting/dex/services/dex.service.ts +++ b/src/subdomains/supporting/dex/services/dex.service.ts @@ -324,7 +324,7 @@ export class DexService { //*** JOBS ***// @DfxCron(CronExpression.EVERY_30_SECONDS, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.DEX_PURCHASE_ORDER, timeout: 1800, }) diff --git a/src/subdomains/supporting/fiat-output/fiat-output-frick.service.ts b/src/subdomains/supporting/fiat-output/fiat-output-frick.service.ts index 01ca34e7f4..3c986daf7f 100644 --- a/src/subdomains/supporting/fiat-output/fiat-output-frick.service.ts +++ b/src/subdomains/supporting/fiat-output/fiat-output-frick.service.ts @@ -28,7 +28,7 @@ export class FiatOutputFrickService { private readonly ibanService: IbanService, ) {} - @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, process: Process.FIAT_OUTPUT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.WORKER, process: Process.FIAT_OUTPUT, timeout: 1800 }) async checkFrickOrderStatus(): Promise { if (DisabledProcess(Process.FIAT_OUTPUT_FRICK_STATUS_CHECK)) return; if (!this.frickService.isAvailable()) return; diff --git a/src/subdomains/supporting/fiat-output/fiat-output-job.service.ts b/src/subdomains/supporting/fiat-output/fiat-output-job.service.ts index 3c5f655011..b457e33daf 100644 --- a/src/subdomains/supporting/fiat-output/fiat-output-job.service.ts +++ b/src/subdomains/supporting/fiat-output/fiat-output-job.service.ts @@ -64,7 +64,7 @@ export class FiatOutputJobService { private readonly bankService: BankService, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.FIAT_OUTPUT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.FIAT_OUTPUT, timeout: 1800 }) async fillFiatOutput() { await this.assignBankAccount(); await this.setReadyDate(); @@ -77,7 +77,7 @@ export class FiatOutputJobService { await this.notifyScryptDeposits(); } - @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, process: Process.FIAT_OUTPUT }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.WORKER, process: Process.FIAT_OUTPUT }) async checkOlkypayOrderStatus(): Promise { if (DisabledProcess(Process.FIAT_OUTPUT_OLKYPAY_STATUS_CHECK)) return; if (!this.olkypayService.isAvailable()) return; @@ -103,7 +103,7 @@ export class FiatOutputJobService { } } - @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, process: Process.FIAT_OUTPUT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.WORKER, process: Process.FIAT_OUTPUT, timeout: 1800 }) async generateReports() { const entities = await this.fiatOutputRepo.find({ where: { reportCreated: false, isComplete: true }, diff --git a/src/subdomains/supporting/fiat-payin/services/fiat-payin-sync.service.ts b/src/subdomains/supporting/fiat-payin/services/fiat-payin-sync.service.ts index a4b3897f8b..188c4c5793 100644 --- a/src/subdomains/supporting/fiat-payin/services/fiat-payin-sync.service.ts +++ b/src/subdomains/supporting/fiat-payin/services/fiat-payin-sync.service.ts @@ -32,7 +32,7 @@ export class FiatPayInSyncService { // --- JOBS --- // - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.FIAT_PAY_IN, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.FIAT_PAY_IN, timeout: 1800 }) async syncCheckout() { if (!this.checkoutService.isAvailable()) { if (!this.unavailableWarningLogged) { diff --git a/src/subdomains/supporting/log/log-job.service.ts b/src/subdomains/supporting/log/log-job.service.ts index d652163225..63b1ee8e91 100644 --- a/src/subdomains/supporting/log/log-job.service.ts +++ b/src/subdomains/supporting/log/log-job.service.ts @@ -112,7 +112,7 @@ export class LogJobService { private readonly paymentBalanceService: PaymentBalanceService, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.TRADING_LOG, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.TRADING_LOG, timeout: 1800 }) async saveTradingLog() { try { // trading log diff --git a/src/subdomains/supporting/log/log.service.ts b/src/subdomains/supporting/log/log.service.ts index fbfc4c680b..1b9545d08d 100644 --- a/src/subdomains/supporting/log/log.service.ts +++ b/src/subdomains/supporting/log/log.service.ts @@ -24,7 +24,7 @@ export class LogService { private readonly settingService: SettingService, ) {} - @DfxCron(CronExpression.EVERY_DAY_AT_11PM, { scope: CronScope.Worker, process: Process.LOG_CLEANUP }) + @DfxCron(CronExpression.EVERY_DAY_AT_11PM, { scope: CronScope.WORKER, process: Process.LOG_CLEANUP }) async cleanup(): Promise { const logCleanupSettings = await this.settingService.getObj('logCleanup', []); diff --git a/src/subdomains/supporting/notification/services/notification-job.service.ts b/src/subdomains/supporting/notification/services/notification-job.service.ts index c4fdf47167..68debf3fe0 100644 --- a/src/subdomains/supporting/notification/services/notification-job.service.ts +++ b/src/subdomains/supporting/notification/services/notification-job.service.ts @@ -33,7 +33,7 @@ export class NotificationJobService { private readonly mailService: MailService, ) {} - @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.MAIL_RETRY, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.WORKER, process: Process.MAIL_RETRY, timeout: 7200 }) async resendUncompletedMails(): Promise { const uncompletedMails = await this.notificationRepo.find({ where: { isComplete: false, created: LessThanOrEqual(Util.minutesBefore(1)) }, diff --git a/src/subdomains/supporting/payin/services/payin-notification.service.ts b/src/subdomains/supporting/payin/services/payin-notification.service.ts index 63202e090b..a7ebb21cd4 100644 --- a/src/subdomains/supporting/payin/services/payin-notification.service.ts +++ b/src/subdomains/supporting/payin/services/payin-notification.service.ts @@ -23,7 +23,7 @@ export class PayInNotificationService { private readonly notificationService: NotificationService, ) {} - @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.Worker, process: Process.PAY_IN_MAIL, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.WORKER, process: Process.PAY_IN_MAIL, timeout: 1800 }) async sendNotificationMails(): Promise { await this.returnedCryptoInput(); } diff --git a/src/subdomains/supporting/payin/services/payin.service.ts b/src/subdomains/supporting/payin/services/payin.service.ts index 223ede1b2b..b9c7440323 100644 --- a/src/subdomains/supporting/payin/services/payin.service.ts +++ b/src/subdomains/supporting/payin/services/payin.service.ts @@ -326,26 +326,26 @@ export class PayInService { // --- JOBS --- // - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.PAY_IN, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.PAY_IN, timeout: 7200 }) async forwardPayInEntries(): Promise { await this.forwardPayIns(); await this.processStrandedSendingPayIns(); } - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.PAY_IN, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.PAY_IN, timeout: 7200 }) async returnPayInEntries(): Promise { await this.returnPayIns(); await this.processStrandedSendingPayIns(); } - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.PAY_IN, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.PAY_IN, timeout: 7200 }) async checkConfirmations(): Promise { await this.checkInputConfirmations(); await this.checkOutputConfirmations(); await this.checkReturnConfirmations(); } - @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.Worker, process: Process.PAY_IN, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.WORKER, process: Process.PAY_IN, timeout: 7200 }) async updateFailedPayments(): Promise { const checkDate = Util.minutesBefore(15); diff --git a/src/subdomains/supporting/payin/strategies/register/impl/base/citrea.strategy.ts b/src/subdomains/supporting/payin/strategies/register/impl/base/citrea.strategy.ts index e1cfefe2e6..3b2bc06aee 100644 --- a/src/subdomains/supporting/payin/strategies/register/impl/base/citrea.strategy.ts +++ b/src/subdomains/supporting/payin/strategies/register/impl/base/citrea.strategy.ts @@ -37,7 +37,7 @@ export abstract class CitreaBaseStrategy extends RegisterStrategy { } // --- JOBS --- // - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.PAY_IN, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.PAY_IN, timeout: 7200 }) async checkPayInEntries(): Promise { const activeDepositAddresses = await this.transactionRequestService.getActiveDepositAddresses( Util.hoursBefore(1), diff --git a/src/subdomains/supporting/payin/strategies/register/impl/bitcoin.strategy.ts b/src/subdomains/supporting/payin/strategies/register/impl/bitcoin.strategy.ts index b738facde7..d04dcdad53 100644 --- a/src/subdomains/supporting/payin/strategies/register/impl/bitcoin.strategy.ts +++ b/src/subdomains/supporting/payin/strategies/register/impl/bitcoin.strategy.ts @@ -31,7 +31,7 @@ export class BitcoinStrategy extends PollingStrategy { } //*** JOBS ***// - @DfxCron(CronExpression.EVERY_SECOND, { scope: CronScope.Worker, process: Process.PAY_IN, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_SECOND, { scope: CronScope.WORKER, process: Process.PAY_IN, timeout: 7200 }) async checkPayInEntries(): Promise { if (!this.payInBitcoinService.isAvailable()) { if (!this.unavailableWarningLogged) { diff --git a/src/subdomains/supporting/payin/strategies/register/impl/cardano.strategy.ts b/src/subdomains/supporting/payin/strategies/register/impl/cardano.strategy.ts index 2bd4a9abee..65f9dc4e84 100644 --- a/src/subdomains/supporting/payin/strategies/register/impl/cardano.strategy.ts +++ b/src/subdomains/supporting/payin/strategies/register/impl/cardano.strategy.ts @@ -41,7 +41,7 @@ export class CardanoStrategy extends RegisterStrategy { } //*** JOBS ***// - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.PAY_IN, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.PAY_IN, timeout: 7200 }) async checkPayInEntries(): Promise { // not configured (no Tatum API key) -> skip, warn once if (!this.payInCardanoService.isConfigured) { diff --git a/src/subdomains/supporting/payin/strategies/register/impl/firo.strategy.ts b/src/subdomains/supporting/payin/strategies/register/impl/firo.strategy.ts index 8c9c545783..8738dee313 100644 --- a/src/subdomains/supporting/payin/strategies/register/impl/firo.strategy.ts +++ b/src/subdomains/supporting/payin/strategies/register/impl/firo.strategy.ts @@ -31,7 +31,7 @@ export class FiroStrategy extends PollingStrategy { } //*** JOBS ***// - @DfxCron(CronExpression.EVERY_SECOND, { scope: CronScope.Worker, process: Process.PAY_IN, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_SECOND, { scope: CronScope.WORKER, process: Process.PAY_IN, timeout: 7200 }) async checkPayInEntries(): Promise { if (!this.payInFiroService.isAvailable()) { if (!this.unavailableWarningLogged) { diff --git a/src/subdomains/supporting/payin/strategies/register/impl/icp.strategy.ts b/src/subdomains/supporting/payin/strategies/register/impl/icp.strategy.ts index 820e3ea606..4fcb631fcf 100644 --- a/src/subdomains/supporting/payin/strategies/register/impl/icp.strategy.ts +++ b/src/subdomains/supporting/payin/strategies/register/impl/icp.strategy.ts @@ -40,7 +40,7 @@ export class InternetComputerStrategy extends RegisterStrategy { } //*** JOBS ***// - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.PAY_IN, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.PAY_IN, timeout: 7200 }) async checkPayInEntries(): Promise { const allDeposits = await this.depositService.getUsedDepositsByBlockchain(this.blockchain); const allDepositAddresses = allDeposits.map((d) => d.address); diff --git a/src/subdomains/supporting/payin/strategies/register/impl/monero.strategy.ts b/src/subdomains/supporting/payin/strategies/register/impl/monero.strategy.ts index 745fb6ff95..3b957e53ca 100644 --- a/src/subdomains/supporting/payin/strategies/register/impl/monero.strategy.ts +++ b/src/subdomains/supporting/payin/strategies/register/impl/monero.strategy.ts @@ -26,7 +26,7 @@ export class MoneroStrategy extends PollingStrategy { } //*** JOBS ***// - @DfxCron(CronExpression.EVERY_SECOND, { scope: CronScope.Worker, process: Process.PAY_IN, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_SECOND, { scope: CronScope.WORKER, process: Process.PAY_IN, timeout: 7200 }) async checkPayInEntries(): Promise { return super.checkPayInEntries(); } diff --git a/src/subdomains/supporting/payin/strategies/register/impl/zano.strategy.ts b/src/subdomains/supporting/payin/strategies/register/impl/zano.strategy.ts index e50eafce08..976018ff00 100644 --- a/src/subdomains/supporting/payin/strategies/register/impl/zano.strategy.ts +++ b/src/subdomains/supporting/payin/strategies/register/impl/zano.strategy.ts @@ -28,7 +28,7 @@ export class ZanoStrategy extends PollingStrategy { } //*** JOBS ***// - @DfxCron(CronExpression.EVERY_SECOND, { scope: CronScope.Worker, process: Process.PAY_IN, timeout: 7200 }) + @DfxCron(CronExpression.EVERY_SECOND, { scope: CronScope.WORKER, process: Process.PAY_IN, timeout: 7200 }) async checkPayInEntries(): Promise { return super.checkPayInEntries(); } diff --git a/src/subdomains/supporting/payment/services/fee.service.ts b/src/subdomains/supporting/payment/services/fee.service.ts index c68ca84f5a..4da3ca1fea 100644 --- a/src/subdomains/supporting/payment/services/fee.service.ts +++ b/src/subdomains/supporting/payment/services/fee.service.ts @@ -89,7 +89,7 @@ export class FeeService { // --- JOBS --- // @DfxCron(CronExpression.EVERY_10_MINUTES, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.BLOCKCHAIN_FEE_UPDATE, timeout: 1800, }) diff --git a/src/subdomains/supporting/payment/services/transaction-helper.ts b/src/subdomains/supporting/payment/services/transaction-helper.ts index 42f1ab4e8e..f1370577d8 100644 --- a/src/subdomains/supporting/payment/services/transaction-helper.ts +++ b/src/subdomains/supporting/payment/services/transaction-helper.ts @@ -87,7 +87,7 @@ export class TransactionHelper implements OnModuleInit { void this.updateCache(); } - @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.Both }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.BOTH }) async updateCache() { this.transactionSpecifications = await this.specRepo.find(); } diff --git a/src/subdomains/supporting/payment/services/transaction-notification.service.ts b/src/subdomains/supporting/payment/services/transaction-notification.service.ts index 60c17f4509..0b63c82a04 100644 --- a/src/subdomains/supporting/payment/services/transaction-notification.service.ts +++ b/src/subdomains/supporting/payment/services/transaction-notification.service.ts @@ -27,7 +27,7 @@ export class TransactionNotificationService { private readonly bankTxService: BankTxService, ) {} - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.TX_MAIL, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.TX_MAIL, timeout: 1800 }) async sendNotificationMails(): Promise { await this.txAssigned(); if (!DisabledProcess(Process.TX_UNASSIGNED_MAIL)) await this.txUnassigned(); diff --git a/src/subdomains/supporting/payment/services/transaction-request.service.ts b/src/subdomains/supporting/payment/services/transaction-request.service.ts index 5a981a5b15..162e962f2e 100644 --- a/src/subdomains/supporting/payment/services/transaction-request.service.ts +++ b/src/subdomains/supporting/payment/services/transaction-request.service.ts @@ -53,7 +53,7 @@ export class TransactionRequestService { // starts by default, which for a minute-based expression spreads them over up to 30 seconds - // moving it here would change when it runs, not just how it is registered. @DfxCron(CronExpression.EVERY_MINUTE, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.TX_REQUEST, useDelay: false, timeout: 7200, @@ -64,7 +64,7 @@ export class TransactionRequestService { } @DfxCron(CronExpression.EVERY_DAY_AT_3AM, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.TX_REQUEST_WAITING_EXPIRY, timeout: 7200, }) diff --git a/src/subdomains/supporting/payout/services/payout.service.ts b/src/subdomains/supporting/payout/services/payout.service.ts index d49017d9a7..87588ccc46 100644 --- a/src/subdomains/supporting/payout/services/payout.service.ts +++ b/src/subdomains/supporting/payout/services/payout.service.ts @@ -173,7 +173,7 @@ export class PayoutService { } //*** JOBS ***// - @DfxCron(CronExpression.EVERY_30_SECONDS, { scope: CronScope.Worker, process: Process.PAY_OUT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_30_SECONDS, { scope: CronScope.WORKER, process: Process.PAY_OUT, timeout: 1800 }) async processOrders(): Promise { await this.checkExistingOrders(); await this.prepareNewOrders(); diff --git a/src/subdomains/supporting/pricing/services/asset-prices-job.service.ts b/src/subdomains/supporting/pricing/services/asset-prices-job.service.ts index c61a94fdbd..9ac4e3e875 100644 --- a/src/subdomains/supporting/pricing/services/asset-prices-job.service.ts +++ b/src/subdomains/supporting/pricing/services/asset-prices-job.service.ts @@ -24,7 +24,7 @@ export class AssetPricesJobService { private readonly assetPriceRepo: AssetPriceRepository, ) {} - @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, process: Process.PRICING, timeout: 3600 }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.WORKER, process: Process.PRICING, timeout: 3600 }) async updatePrices() { const assetsToUpdate = await this.assetService.getPricedAssets(); const updates: UpdateResult[] = []; @@ -59,7 +59,7 @@ export class AssetPricesJobService { await this.assetService.updateAssets(updates); } - @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.Worker, process: Process.PRICING, timeout: 3600 }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.WORKER, process: Process.PRICING, timeout: 3600 }) async updatePaymentPrices() { const relevantFiats = await this.fiatService.getActiveFiat(); const relevantAssets = await this.assetService.getPaymentAssets(); diff --git a/src/subdomains/supporting/pricing/services/fiat-prices.service.ts b/src/subdomains/supporting/pricing/services/fiat-prices.service.ts index 3cf424739a..d030f8a5fa 100644 --- a/src/subdomains/supporting/pricing/services/fiat-prices.service.ts +++ b/src/subdomains/supporting/pricing/services/fiat-prices.service.ts @@ -16,7 +16,7 @@ export class FiatPricesService { ) {} // --- JOBS --- // - @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, process: Process.PRICING, timeout: 3600 }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.WORKER, process: Process.PRICING, timeout: 3600 }) async updatePrices() { const fiats = await this.fiatService.getActiveFiat(); diff --git a/src/subdomains/supporting/realunit/realunit-job.service.ts b/src/subdomains/supporting/realunit/realunit-job.service.ts index 1842df939e..515e5b411a 100644 --- a/src/subdomains/supporting/realunit/realunit-job.service.ts +++ b/src/subdomains/supporting/realunit/realunit-job.service.ts @@ -22,7 +22,7 @@ export class RealUnitJobService { // triggered outside the DFX payment flow (e.g. booked manually by the issuer) would otherwise // leave the quote in WaitingForPayment and keep showing a pending payment to the customer. @DfxCron(CronExpression.EVERY_MINUTE, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.REALUNIT_QUOTE_COMPLETION, timeout: 1800, }) @@ -92,7 +92,7 @@ export class RealUnitJobService { // atomic claim and the broadcast/callback in confirmTransfer — see // RealUnitService.reconcilePendingTransfers for the actual reconciliation logic. @DfxCron(CronExpression.EVERY_5_MINUTES, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.REALUNIT_TRANSFER_RECONCILIATION, timeout: 1800, }) diff --git a/src/subdomains/supporting/support-issue/services/limit-request-notification.service.ts b/src/subdomains/supporting/support-issue/services/limit-request-notification.service.ts index 74d1719aa3..a97b66a5d8 100644 --- a/src/subdomains/supporting/support-issue/services/limit-request-notification.service.ts +++ b/src/subdomains/supporting/support-issue/services/limit-request-notification.service.ts @@ -23,7 +23,7 @@ export class LimitRequestNotificationService { ) {} @DfxCron(CronExpression.EVERY_5_MINUTES, { - scope: CronScope.Worker, + scope: CronScope.WORKER, process: Process.LIMIT_REQUEST_MAIL, timeout: 1800, }) diff --git a/src/subdomains/supporting/support-issue/services/support-escalation.service.ts b/src/subdomains/supporting/support-issue/services/support-escalation.service.ts index 6fb6d6c89a..0f9fac833a 100644 --- a/src/subdomains/supporting/support-issue/services/support-escalation.service.ts +++ b/src/subdomains/supporting/support-issue/services/support-escalation.service.ts @@ -200,7 +200,7 @@ export class SupportEscalationService { // --- Escalation detection --- - @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.Worker, process: Process.SUPPORT_BOT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.WORKER, process: Process.SUPPORT_BOT, timeout: 1800 }) async checkEscalations(): Promise { if (!this.token) return; diff --git a/src/subdomains/supporting/support-issue/services/support-issue-job.service.ts b/src/subdomains/supporting/support-issue/services/support-issue-job.service.ts index 39ec3ae8f2..53c500c419 100644 --- a/src/subdomains/supporting/support-issue/services/support-issue-job.service.ts +++ b/src/subdomains/supporting/support-issue/services/support-issue-job.service.ts @@ -32,7 +32,7 @@ export class SupportIssueJobService { private readonly settingsService: SettingService, ) {} - @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Worker, process: Process.SUPPORT_BOT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.WORKER, process: Process.SUPPORT_BOT, timeout: 1800 }) async autoOnHold() { const entities = await this.supportIssueRepo.find({ where: { @@ -52,7 +52,7 @@ export class SupportIssueJobService { } } - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.SUPPORT_BOT, timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.SUPPORT_BOT, timeout: 1800 }) async sendAutoResponses() { const disabledTemplates = await this.settingsService .get('supportBot') diff --git a/src/tracing.ts b/src/tracing.ts index d90b653d60..760703c567 100644 --- a/src/tracing.ts +++ b/src/tracing.ts @@ -96,9 +96,9 @@ export function isTelemetryEnabled(): boolean { } /** - * The name both processes report to the collector. Without the distinction they would appear as - * one service, and any panel not restricted to server spans would mix the worker's outgoing calls - * with the request traffic. + * The name both processes report to the collector. Without the distinction they would report as + * one service, and a consumer of the traces could not tell the worker's outgoing calls apart from + * the calls a request made. * * Reads the environment directly rather than the configuration: tracing starts before anything * else, and importing the configuration here would pull in the instrumented modules before the From 3367ccc953386387b4996bb9e057503c98ac714f Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:41 +0200 Subject: [PATCH 24/86] Close the two remaining ways the monitoring write can lose a value The lock made writers wait for each other, but two cases survived it. A writer that waited on the lock holds a value from before that wait. Writing it regardless puts an older measurement back over a newer one, and it stays there until the metric changes again in that process - the same shape of lost update the lock was meant to close, one step later. Each changed metric is now only taken over if its timestamp is not older than the stored one. And an absent row cannot be locked. On a database where the row does not exist yet, two writers reach the insert together and one loses on the primary key; its change is then dropped, because the next comparison is against its own previous state and finds nothing changed. The merge is retried once, which finds the row the other writer created. The balance store is filled once at start-up as well. The endpoint answers from the store alone, so without it the window after a restart is a full interval wide, and waiting does not shorten it. Tests cover all three: the older value is not written back, the retry recovers the insert conflict without notifying, and start-up fills the store. Also restores CRLF line endings in two files that tooling had converted to LF, which turned small edits into whole-file diffs. --- src/config/__tests__/cron-role.config.spec.ts | 8 +- src/config/config.ts | 4 +- .../spark/__tests__/spark-client.spec.ts | 11 +- .../blockchain/spark/spark-client.ts | 6 +- .../controllers/exchange.controller.ts | 6 +- src/shared/services/dfx-cron.service.ts | 7 +- src/shared/utils/cron.ts | 141 +++++++++--------- .../controllers/transaction.controller.ts | 2 +- .../__tests__/monitoring.service.spec.ts | 45 ++++++ .../monitor-connection-pool.service.ts | 96 ++++++------ .../core/monitoring/monitoring.service.ts | 66 ++++---- .../services/payment-cron.service.ts | 10 +- .../dashboard-financial.service.spec.ts | 22 ++- .../dashboard/dashboard-financial.service.ts | 32 ++-- src/tracing.ts | 11 +- 15 files changed, 269 insertions(+), 198 deletions(-) diff --git a/src/config/__tests__/cron-role.config.spec.ts b/src/config/__tests__/cron-role.config.spec.ts index a4e3c37e95..eaebf978b7 100644 --- a/src/config/__tests__/cron-role.config.spec.ts +++ b/src/config/__tests__/cron-role.config.spec.ts @@ -10,11 +10,9 @@ describe('parseCronRole', () => { it.each([undefined, '', ' ', 'All', 'WORKER', 'api ', 'true', 'none'])( 'throws on %p instead of picking a role', (value) => { - // Every possible default is silent in one direction: 'worker' would make a misconfigured - // API process run all background work a second time, 'api' would make a misconfigured - // worker do nothing at all. Neither raises an error, and duplicate execution of financial - // jobs is worse than a failed boot. The empty string is included deliberately — a - // `CRON_ROLE=` line or an unresolved `${VAR}` is the likeliest accident of all. + // Verifies that a missing, empty or unknown value is rejected rather than mapped to a + // default: `parseCronRole` has no fallback branch, and the empty string takes the same path + // as any other invalid value. expect(() => parseCronRole(value)).toThrow(/expected one of all, api, worker/); }, ); diff --git a/src/config/config.ts b/src/config/config.ts index 5be64fc114..7c142ced99 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -1538,8 +1538,8 @@ export class Configuration { * worker do nothing at all. Neither produces an error, and duplicate execution of financial * jobs is far more damaging than a failed boot. * - * The empty string is rejected for the same reason, even though it is the more likely accident - * — a `CRON_ROLE=` line in an env file, or an unresolved `${VAR}`. + * The empty string is rejected for the same reason: a `CRON_ROLE=` line in an env file or an + * unresolved `${VAR}` both arrive here as one. * * `all` is not a convenience value but the single-process mode: one process runs every job, * which is what local development, the test suite and any environment without a separate worker diff --git a/src/integration/blockchain/spark/__tests__/spark-client.spec.ts b/src/integration/blockchain/spark/__tests__/spark-client.spec.ts index ee3d218dfd..5d6808b2ba 100644 --- a/src/integration/blockchain/spark/__tests__/spark-client.spec.ts +++ b/src/integration/blockchain/spark/__tests__/spark-client.spec.ts @@ -79,9 +79,8 @@ describe('SparkClient', () => { describe('token optimization timer', () => { it('runs the wallet maintenance in the worker process', () => { - // On-chain wallet maintenance is global work and must run in exactly one - // process. The timer predates the scheduler and bypasses it, so it carries - // the role check itself. + // The timer is not registered through the scheduler, so the client decides itself: under + // the worker role it is created, with the five-minute interval the client sets. const interval = jest.spyOn(global, 'setInterval'); new SparkClient(); @@ -93,8 +92,7 @@ describe('SparkClient', () => { }); it('runs it in the single-process role', () => { - // The mode every environment without a separate worker runs in, and the one the claim - // "behaviour is unchanged" rests on. Only the API role may skip this timer. + // The single-process role must keep the timer: only CronRole.API skips it. mockCronRole = 'all'; const interval = jest.spyOn(global, 'setInterval'); @@ -106,8 +104,7 @@ describe('SparkClient', () => { }); it('does not run it in the API process', () => { - // Both processes run the same image against the same seed: without this - // check the maintenance would run twice, unsynchronised. + // The counterpart: under the API role no timer is created at all. mockCronRole = 'api'; const interval = jest.spyOn(global, 'setInterval'); diff --git a/src/integration/blockchain/spark/spark-client.ts b/src/integration/blockchain/spark/spark-client.ts index 7a759569fb..b20444a454 100644 --- a/src/integration/blockchain/spark/spark-client.ts +++ b/src/integration/blockchain/spark/spark-client.ts @@ -226,9 +226,9 @@ export class SparkClient extends BlockchainClient { } private startTokenOptimization(): void { - // On-chain wallet maintenance is global work: it must run in exactly one process. This - // timer predates the scheduler and bypasses it, so without the role check the API process - // would drive optimizeTokenOutputs against the same seed as the worker. + // On-chain wallet maintenance belongs to exactly one process. This timer is not registered + // through DfxCronService, so it carries the role check itself: under CronRole.API no timer + // is created and optimizeTokenOutputs is never called from here. if (GetConfig().cronRole === CronRole.API) return; if (this.tokenOptimizationInterval) clearInterval(this.tokenOptimizationInterval); diff --git a/src/integration/exchange/controllers/exchange.controller.ts b/src/integration/exchange/controllers/exchange.controller.ts index bed51491b6..a8d766b2c3 100644 --- a/src/integration/exchange/controllers/exchange.controller.ts +++ b/src/integration/exchange/controllers/exchange.controller.ts @@ -171,9 +171,9 @@ export class ExchangeController { } // --- JOBS --- // - // Api, not Both: `trades` is filled by POST :exchange/trade and read by GET trade/:id, both - // request paths of this controller. Nothing outside a request touches it, so in a process - // without ingress the map stays empty and the job has nothing to do. + // Api, not Both: `trades` is filled by POST :exchange/trade and read by GET trade/:id, the + // request paths of this controller shown below. In a process those requests never reach, the + // map stays empty and this job has nothing to work on. @DfxCron(CronExpression.EVERY_30_SECONDS, { scope: CronScope.API, timeout: 1800 }) async checkTrades() { const openTrades = Object.values(this.trades).filter(({ status }) => status === TradeStatus.OPEN); diff --git a/src/shared/services/dfx-cron.service.ts b/src/shared/services/dfx-cron.service.ts index 0a3fa9edf4..6de480ca0f 100644 --- a/src/shared/services/dfx-cron.service.ts +++ b/src/shared/services/dfx-cron.service.ts @@ -60,9 +60,10 @@ export class DfxCronService implements OnModuleInit { }); }); - // The effective split is read from this line, not inferred from a table: a job registered - // through a dynamically resolved provider or an abstract base class is counted here and - // nowhere else. It stays on `info` so it is findable without changing the log level. + // Counts what this process actually registered, including jobs reaching the scheduler + // through a dynamically resolved provider or an abstract base class - the discovery above + // sees those, a list of decorators would not. Stays on `info` so the split is readable + // without changing the log level. const total = registered.length + skipped; const byScope = Object.values(CronScope) .map((scope) => `${scope}: ${registered.filter((s) => s === scope).length}`) diff --git a/src/shared/utils/cron.ts b/src/shared/utils/cron.ts index 5895b48127..ceda9f8fdc 100644 --- a/src/shared/utils/cron.ts +++ b/src/shared/utils/cron.ts @@ -1,71 +1,70 @@ -import { CronExpression } from '@nestjs/schedule'; -import { Process } from '../services/process.service'; -import { CustomCronExpression } from './custom-cron-expression'; - -/** - * Which process a job belongs to. - * - * The distinction is not about importance but about where a job's effect is visible. Most jobs - * only touch the database or an external system, so any process can run them and exactly one - * should. The exceptions are jobs maintaining state that lives inside the process itself - a - * class field, a module-global variable, a Node process metric - because such state is only - * useful in the process whose requests read it. - */ -export enum CronScope { - /** Worker process only. The normal case: anything writing to the database or driving business forward. */ - WORKER = 'worker', - /** - * API process only. Maintains or measures state read exclusively from a request path, or - * drives work bound to the connections that process holds open. - */ - API = 'api', - /** - * Every process. Maintains or measures process-local state that both sides read. - * - * Running such a job twice must be harmless by construction: refreshing an in-memory copy of - * global state, expiring a local cache or writing a log line qualifies. Writing to the - * database or driving business forward does not - cron locks are per-process and cannot - * prevent duplicate execution across processes. - */ - BOTH = 'both', -} - -export interface DfxCronOptParams { - process?: Process; - useDelay?: boolean; - timeout?: number; -} - -/** - * Parameters of a cron job. `scope` is mandatory and has no default. - * - * A wrong classification fails silently - a job wrongly scoped `worker` leaves the cache it - * maintains empty in the process that reads it, with no error anywhere. A default plus a list - * of exceptions moves that decision into a hand-maintained list the compiler never sees, and - * such a list grows and goes stale; pinning it in a test proves the state of the list, not the - * property it stands for. Requiring the field puts the question in front of whoever adds a job, - * which is the only check that stays complete as jobs are added. - */ -export interface DfxCronRequiredParams extends DfxCronOptParams { - scope: CronScope; -} - -export type DfxCronExpression = CronExpression | CustomCronExpression; - -export interface DfxCronParams extends DfxCronRequiredParams { - expression: DfxCronExpression; -} - -export const DFX_CRONJOB_PARAMS = 'DFXCronjobParams'; - -export function DfxCron(expression: DfxCronExpression, required: DfxCronRequiredParams) { - return function (target: any, propertyKey: string, descriptor: PropertyDescriptor) { - const methodRef = target[propertyKey]; - - const params: DfxCronParams = { expression, ...required }; - - Reflect.defineMetadata(DFX_CRONJOB_PARAMS, params, methodRef); - - return descriptor; - }; -} +import { CronExpression } from '@nestjs/schedule'; +import { Process } from '../services/process.service'; +import { CustomCronExpression } from './custom-cron-expression'; + +/** + * Which process a job belongs to. + * + * The distinction is not about importance but about where a job's effect is visible. Most jobs + * only touch the database or an external system, so any process can run them and exactly one + * should. The exceptions are jobs maintaining state that lives inside the process itself - a + * class field, a module-global variable, a Node process metric - because such state is only + * useful in the process whose requests read it. + */ +export enum CronScope { + /** Worker process only. The normal case: anything writing to the database or driving business forward. */ + WORKER = 'worker', + /** + * API process only. Maintains or measures state read exclusively from a request path, or + * drives work bound to the connections that process holds open. + */ + API = 'api', + /** + * Every process. Maintains or measures process-local state that both sides read. + * + * Running such a job twice must be harmless by construction: refreshing an in-memory copy of + * global state, expiring a local cache or writing a log line qualifies. Writing to the + * database or driving business forward does not - cron locks are per-process and cannot + * prevent duplicate execution across processes. + */ + BOTH = 'both', +} + +export interface DfxCronOptParams { + process?: Process; + useDelay?: boolean; + timeout?: number; +} + +/** + * Parameters of a cron job. `scope` is mandatory and has no default. + * + * A wrong classification fails silently - a job wrongly scoped `worker` leaves the cache it + * maintains empty in the process that reads it, with no error anywhere. Requiring the field puts + * that decision in front of whoever adds a job, and the compiler enforces it. A default plus a + * list of exceptions would move the same decision into a list the compiler does not see, where a + * test can only pin the entries it already has. + */ +export interface DfxCronRequiredParams extends DfxCronOptParams { + scope: CronScope; +} + +export type DfxCronExpression = CronExpression | CustomCronExpression; + +export interface DfxCronParams extends DfxCronRequiredParams { + expression: DfxCronExpression; +} + +export const DFX_CRONJOB_PARAMS = 'DFXCronjobParams'; + +export function DfxCron(expression: DfxCronExpression, required: DfxCronRequiredParams): MethodDecorator { + return function (target: object, propertyKey: string | symbol, descriptor: PropertyDescriptor) { + const methodRef = target[propertyKey]; + + const params: DfxCronParams = { expression, ...required }; + + Reflect.defineMetadata(DFX_CRONJOB_PARAMS, params, methodRef); + + return descriptor; + }; +} diff --git a/src/subdomains/core/history/controllers/transaction.controller.ts b/src/subdomains/core/history/controllers/transaction.controller.ts index 6e38cbf18d..d285cf28a5 100644 --- a/src/subdomains/core/history/controllers/transaction.controller.ts +++ b/src/subdomains/core/history/controllers/transaction.controller.ts @@ -117,7 +117,7 @@ export class TransactionController { // --- JOBS --- // // Api, not Both: `refundList` is filled and read by the refund request paths of this - // controller only, so it stays empty in a process without ingress. + // controller, so it stays empty in a process those requests never reach. @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.API }) checkLists() { for (const [key, refundData] of this.refundList.entries()) { diff --git a/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts b/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts index 98d7989045..0d39a06f9e 100644 --- a/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts +++ b/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts @@ -151,6 +151,51 @@ describe('MonitoringService', () => { expect(written).toHaveLength(1); }); + it('does not put an older measurement back over a newer one', async () => { + // This process may have waited on the lock while another wrote a later measurement of the + // same metric. Writing regardless would restore the older value, and it would stay until + // the metric changes here again. + const prev: SystemState = { bank: { balance: metric({ chf: 1 }, '2020-01-01T00:00:00Z') } }; + const stale: SystemState = { bank: { balance: metric({ chf: 2 }, '2020-01-01T00:05:00Z') } }; + + await service['persist'](prev, stale); + + const result = JSON.parse(written[0].data) as SystemState; + + expect(result.bank.balance.data).toEqual({ chf: 42 }); + }); + + it('retries when the row cannot be locked because it does not exist yet', async () => { + // An absent row cannot be locked, so two writers can reach the insert together and one + // loses on the primary key. Without the retry that process's change is dropped until the + // metric happens to change again. + let attempts = 0; + Object.defineProperty(repo, 'manager', { + value: { + transaction: (run: (m: unknown) => Promise) => { + attempts++; + if (attempts === 1) return Promise.reject(new Error('duplicate key value violates unique constraint')); + return run({ + findOne: jest.fn().mockResolvedValue(snapshot(persisted)), + save: jest.fn().mockImplementation((_e: unknown, row: { id: number; data: string }) => { + written.push(row); + return Promise.resolve(row); + }), + }); + }, + }, + configurable: true, + }); + + const next: SystemState = { ledger: { open: metric({ count: 1 }, '2020-01-01T00:20:00Z') } }; + + await service['persist']({}, next); + + expect(attempts).toBe(2); + expect(JSON.parse(written[0].data).ledger.open.data).toEqual({ count: 1 }); + expect(notificationService.sendMail).not.toHaveBeenCalled(); + }); + it('writes a metric that did not exist before', async () => { const next: SystemState = { ledger: { open: metric({ count: 3 }, '2020-01-01T00:20:00Z') } }; diff --git a/src/subdomains/core/monitoring/monitor-connection-pool.service.ts b/src/subdomains/core/monitoring/monitor-connection-pool.service.ts index b21d687156..a5c39b614b 100644 --- a/src/subdomains/core/monitoring/monitor-connection-pool.service.ts +++ b/src/subdomains/core/monitoring/monitor-connection-pool.service.ts @@ -1,48 +1,48 @@ -import { Injectable } from '@nestjs/common'; -import { CronExpression } from '@nestjs/schedule'; -import { Config } from 'src/config/config'; -import { DfxLogger } from 'src/shared/services/dfx-logger'; -import { Process } from 'src/shared/services/process.service'; -import { CronScope, DfxCron } from 'src/shared/utils/cron'; -import { DataSource } from 'typeorm'; -import { PostgresConnectionOptions } from 'typeorm/driver/postgres/PostgresConnectionOptions'; -import { PostgresDriver } from 'typeorm/driver/postgres/PostgresDriver'; - -@Injectable() -export class MonitorConnectionPoolService { - private readonly logger = new DfxLogger(MonitorConnectionPoolService); - - private readonly dbConnectionPool: any; // pg.Pool - - constructor(dataSource: DataSource) { - const dbDriver = dataSource.driver as PostgresDriver; - this.dbConnectionPool = dbDriver.master; - } - - @DfxCron(CronExpression.EVERY_SECOND, { scope: CronScope.BOTH, process: Process.MONITOR_CONNECTION_POOL }) - monitorConnectionPool() { - const dbOptions = Config.database as PostgresConnectionOptions; - const dbMaxPoolConnections = dbOptions.poolSize ?? 10; - - const total = this.dbConnectionPool.totalCount; - const idle = this.dbConnectionPool.idleCount; - const waiting = this.dbConnectionPool.waitingCount; - - if (dbMaxPoolConnections === total && idle === 0) { - // Warning, if there are all connections in use - this.logger.warn(`ConnectionPool with max. borrowed connections: T${total}/I${idle}/W${waiting}`); - } else if (waiting > 0) { - // Info, if there is a pending connection - this.logger.info(`ConnectionPool with pending connections: T${total}/I${idle}/W${waiting}`); - } - } - - @DfxCron(CronExpression.EVERY_10_SECONDS, { scope: CronScope.BOTH, process: Process.MONITOR_CONNECTION_POOL }) - monitorConnectionPoolStatic() { - const total = this.dbConnectionPool.totalCount; - const idle = this.dbConnectionPool.idleCount; - const waiting = this.dbConnectionPool.waitingCount; - - this.logger.info(`ConnectionPool connections: T${total}/I${idle}/W${waiting}`); - } -} +import { Injectable } from '@nestjs/common'; +import { CronExpression } from '@nestjs/schedule'; +import { Config } from 'src/config/config'; +import { DfxLogger } from 'src/shared/services/dfx-logger'; +import { Process } from 'src/shared/services/process.service'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; +import { DataSource } from 'typeorm'; +import { PostgresConnectionOptions } from 'typeorm/driver/postgres/PostgresConnectionOptions'; +import { PostgresDriver } from 'typeorm/driver/postgres/PostgresDriver'; + +@Injectable() +export class MonitorConnectionPoolService { + private readonly logger = new DfxLogger(MonitorConnectionPoolService); + + private readonly dbConnectionPool: any; // pg.Pool + + constructor(dataSource: DataSource) { + const dbDriver = dataSource.driver as PostgresDriver; + this.dbConnectionPool = dbDriver.master; + } + + @DfxCron(CronExpression.EVERY_SECOND, { scope: CronScope.BOTH, process: Process.MONITOR_CONNECTION_POOL }) + monitorConnectionPool() { + const dbOptions = Config.database as PostgresConnectionOptions; + const dbMaxPoolConnections = dbOptions.poolSize ?? 10; + + const total = this.dbConnectionPool.totalCount; + const idle = this.dbConnectionPool.idleCount; + const waiting = this.dbConnectionPool.waitingCount; + + if (dbMaxPoolConnections === total && idle === 0) { + // Warning, if there are all connections in use + this.logger.warn(`ConnectionPool with max. borrowed connections: T${total}/I${idle}/W${waiting}`); + } else if (waiting > 0) { + // Info, if there is a pending connection + this.logger.info(`ConnectionPool with pending connections: T${total}/I${idle}/W${waiting}`); + } + } + + @DfxCron(CronExpression.EVERY_10_SECONDS, { scope: CronScope.BOTH, process: Process.MONITOR_CONNECTION_POOL }) + monitorConnectionPoolStatic() { + const total = this.dbConnectionPool.totalCount; + const idle = this.dbConnectionPool.idleCount; + const waiting = this.dbConnectionPool.waitingCount; + + this.logger.info(`ConnectionPool connections: T${total}/I${idle}/W${waiting}`); + } +} diff --git a/src/subdomains/core/monitoring/monitoring.service.ts b/src/subdomains/core/monitoring/monitoring.service.ts index 31e9b890e6..727bb2c093 100644 --- a/src/subdomains/core/monitoring/monitoring.service.ts +++ b/src/subdomains/core/monitoring/monitoring.service.ts @@ -2,6 +2,7 @@ import { Injectable, NotFoundException, OnModuleInit } from '@nestjs/common'; import { cloneDeep, isEqual } from 'lodash'; import { BehaviorSubject, debounceTime, pairwise } from 'rxjs'; import { DfxLogger } from 'src/shared/services/dfx-logger'; +import { Util } from 'src/shared/utils/util'; import { MailContext, MailType } from 'src/subdomains/supporting/notification/enums'; import { NotificationService } from 'src/subdomains/supporting/notification/services/notification.service'; import { MetricObserver } from './metric.observer'; @@ -133,11 +134,10 @@ export class MonitoringService implements OnModuleInit { /** * Writes only the metrics this process changed, merged into the stored state. * - * The whole system state lives in a single row, and every process subscribing to its own - * updates writes it. Replacing the row with this process's view would drop whatever another - * process wrote in the meantime - the API process would overwrite the observers' work with its - * boot state, and the webhook path the other way round. Merging the changed metrics into the - * stored row makes it irrelevant which process writes. + * The whole system state lives in a single row (`id: 1`), and `initState` above subscribes + * this writer in whichever process the service is instantiated in. Replacing the row with this + * instance's view would drop metrics another writer put there; merging only the changed ones + * keeps both. * * Read, merge and write happen inside one transaction that locks the row. Without the lock the * merge only narrows the race instead of closing it: two writers that read before either wrote @@ -150,23 +150,11 @@ export class MonitoringService implements OnModuleInit { const changed = this.changedMetrics(prevState, newState); if (!changed.length) return; - const merged = await this.systemStateSnapshotRepo.manager.transaction(async (manager) => { - const row = await manager.findOne(SystemStateSnapshot, { - where: { id: 1 }, - lock: { mode: 'pessimistic_write' }, - }); - - const stored: SystemState = row ? JSON.parse(row.data) : {}; - const state = cloneDeep(stored); - - for (const [subsystem, metric] of changed) { - state[subsystem] = { ...(state[subsystem] ?? {}), [metric]: newState[subsystem][metric] }; - } - - await manager.save(SystemStateSnapshot, { id: 1, data: JSON.stringify(state) }); - - return state; - }); + // A second attempt covers the case where the row does not exist yet: it cannot be locked, + // so two writers can reach the insert together and one loses on the primary key. Retrying + // finds the row the other one created and merges into it, instead of dropping this + // process's change until the metric happens to change again. + const merged = await Util.retry(() => this.mergeIntoStoredState(changed, newState), 2); this.#storedState = { state: merged, loaded: Date.now() }; } catch (e) { @@ -180,6 +168,34 @@ export class MonitoringService implements OnModuleInit { } } + private async mergeIntoStoredState(changed: [string, string][], newState: SystemState): Promise { + return this.systemStateSnapshotRepo.manager.transaction(async (manager) => { + const row = await manager.findOne(SystemStateSnapshot, { + where: { id: 1 }, + lock: { mode: 'pessimistic_write' }, + }); + + const stored: SystemState = row ? JSON.parse(row.data) : {}; + const state = cloneDeep(stored); + + for (const [subsystem, metric] of changed) { + const candidate = newState[subsystem][metric]; + + // The value this process holds is not automatically the newer one: it may have waited on + // the lock while another process wrote a later measurement of the same metric. Writing it + // anyway would put the older value back, and it would stay there until the metric changes + // again in this process. + if (this.updatedAt(candidate) < this.updatedAt(state[subsystem]?.[metric])) continue; + + state[subsystem] = { ...(state[subsystem] ?? {}), [metric]: candidate }; + } + + await manager.save(SystemStateSnapshot, { id: 1, data: JSON.stringify(state) }); + + return state; + }); + } + /** The metrics whose data differs between the two states, as [subsystem, metric] pairs. */ private changedMetrics(prevState: SystemState, newState: SystemState): [string, string][] { return Object.entries(newState ?? {}).flatMap(([subsystemName, subsystemState]) => @@ -192,7 +208,7 @@ export class MonitoringService implements OnModuleInit { ); } - /** Reads the persisted state without notifying: this runs on every read, not once at start. */ + /** Reads the persisted state without notifying: unlike loadState, this runs on every read. */ private async readState(): Promise { const latestPersistedState = await this.systemStateSnapshotRepo.findOne({ where: {}, order: { id: 'DESC' } }); @@ -223,8 +239,8 @@ export class MonitoringService implements OnModuleInit { // Concurrent requests share one read rather than each issuing their own. if (!this.#pendingLoad) { const load = this.readState().catch((e) => { - // Deliberately no mail: unlike the load at start-up, this path runs on every request, so - // a database problem would answer itself with a flood of mails. + // No mail here, unlike loadState: this path is reached from getState, so a failing read + // would notify once per request instead of once per start-up. this.logger.error('Failed to read the persisted system state:', e); return null; }); diff --git a/src/subdomains/core/payment-link/services/payment-cron.service.ts b/src/subdomains/core/payment-link/services/payment-cron.service.ts index e134185417..7e3f593822 100644 --- a/src/subdomains/core/payment-link/services/payment-cron.service.ts +++ b/src/subdomains/core/payment-link/services/payment-cron.service.ts @@ -18,11 +18,11 @@ export class PaymentCronService { // Api, not Worker: this job and checkTxConfirmations both end up in // PaymentLinkPaymentService.doSave(), which resolves the AsyncMap that PaymentLinkController's - // waitForPayment is waiting on and pushes the device activation into the RxJS subject the - // gateway delivers to its connected clients. Both mechanisms only reach clients connected to - // the very process the job runs in, which is what CronScope.API expresses. `Both` is no option - // either: the jobs write to the database and trigger merchant webhooks, which two processes - // without a shared lock would do twice. + // waitForPayment awaits and pushes the device activation into the RxJS subject PaymentLinkGateway + // subscribes to. Both hold their state in the instance, so they only reach a caller of this same + // process. `Both` is no option either: the jobs write to the database and trigger merchant + // webhooks, which a second registration would repeat - the lock in DfxCronService is per + // process. @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.API, process: Process.PAYMENT_EXPIRATION }) async processExpiredPayments(): Promise { await this.paymentLinkPaymentService.processExpiredPayments(); diff --git a/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts b/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts index 39fad26132..5d4a725b6b 100644 --- a/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts +++ b/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts @@ -10,6 +10,7 @@ import { FinancialLogSummary } from '../../log/log.repository'; import { LogService } from '../../log/log.service'; import { DashboardFinancialService } from '../dashboard-financial.service'; import { LatestBalanceResponseDto } from '../dto/financial-log.dto'; +import { TestUtil } from 'src/shared/utils/test.util'; import { LatestBalanceStore } from '../latest-balance.store'; describe('DashboardFinancialService', () => { @@ -30,16 +31,14 @@ describe('DashboardFinancialService', () => { { provide: AssetService, useValue: assetService }, { provide: RefRewardService, useValue: createMock() }, { provide: LatestBalanceStore, useValue: latestBalanceStore }, + TestUtil.provideConfig(), ], }).compile(); service = module.get(DashboardFinancialService); }); - /** - * Drives the job the way it runs in production: it reads the most recent FinancialDataLog and - * resolves the assets itself, where the removed write-through was handed both in memory. - */ + /** Mocks the log entry and the assets the job resolves, then runs it. */ async function refreshFrom( timestamp: Date, assetLog: AssetLog, @@ -503,6 +502,21 @@ describe('DashboardFinancialService', () => { expect(latestBalanceStore.set).toHaveBeenCalledWith(expected); }); + it('fills the store at start-up instead of waiting for the first scheduled run', async () => { + // The endpoint answers from the store alone. Without this the window after a restart is a + // full cron interval wide, and it does not shrink by waiting. + jest.spyOn(logService, 'getLatestFinancialLog').mockResolvedValue({ + id: 1, + created: new Date('2026-07-14T12:00:00Z'), + message: JSON.stringify({ assets: {}, balancesByFinancialType: {} }), + } as Log); + + service.onModuleInit(); + await new Promise(process.nextTick); + + expect(latestBalanceStore.set).toHaveBeenCalled(); + }); + it('leaves the store untouched when there is no log entry yet', async () => { jest.spyOn(logService, 'getLatestFinancialLog').mockResolvedValue(undefined); diff --git a/src/subdomains/supporting/dashboard/dashboard-financial.service.ts b/src/subdomains/supporting/dashboard/dashboard-financial.service.ts index bc7a48572c..b9016e745d 100644 --- a/src/subdomains/supporting/dashboard/dashboard-financial.service.ts +++ b/src/subdomains/supporting/dashboard/dashboard-financial.service.ts @@ -1,5 +1,6 @@ -import { Injectable } from '@nestjs/common'; +import { Injectable, OnModuleInit } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; +import { Config, CronRole } from 'src/config/config'; import { Asset } from 'src/shared/models/asset/asset.entity'; import { AssetService } from 'src/shared/models/asset/asset.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; @@ -22,7 +23,7 @@ import { import { LatestBalanceStore } from './latest-balance.store'; @Injectable() -export class DashboardFinancialService { +export class DashboardFinancialService implements OnModuleInit { private readonly logger = new DfxLogger(DashboardFinancialService); constructor( @@ -32,6 +33,14 @@ export class DashboardFinancialService { private readonly latestBalanceStore: LatestBalanceStore, ) {} + onModuleInit() { + // Fills the store once at start-up instead of leaving it empty until the first scheduled run. + // The endpoint answers from the store alone, so without this the window after a restart is a + // full cron interval wide - and it does not shrink by waiting, because the job that used to + // fill the store on the spot is gone. + if (Config.cronRole !== CronRole.WORKER) void this.refreshLatestBalance().catch(() => undefined); + } + async getFinancialLog(from?: Date, dailySample?: boolean, includeByType?: boolean): Promise { // BTC price is projected in SQL and needs btcAssetId as a parameter, so resolve getBtcCoin first. // One extra sequential roundtrip vs the previous Promise.all, judged negligible against the @@ -136,20 +145,13 @@ export class DashboardFinancialService { * GET /v1/dashboard/financial/latest keeps answering from process memory without touching the * database - see getLatestBalance below, which reads the store and nothing else. * - * Scope Api, because the store it fills is process-local and its only reader is that endpoint. - * The expensive part stays where it was: the financial aggregation writing the log entry runs in - * the worker, and this job only reads what that one produced. Parse and aggregation happen once - * a minute outside any request instead of on every call, which is the same work the endpoint did - * before the store existed. - * - * This is not a fallback taken only when the store is empty: such a path would run once per - * deployment and would never be exercised. It runs every minute, in normal operation as much as - * after a failure. + * Scope Api, because the store it fills is a field of this service and its reader is the + * endpoint above. It only reads: LogJobService writes the entry, this parses it and aggregates + * - once a minute, outside any request. * - * The trade-off is one extra read per minute compared to the write-through this replaces, which - * was handed its inputs by the caller. It applies in the single-process role too, where both - * jobs run in the same process - accepted deliberately: sharing state between the two would tie - * the endpoint's cache back to the aggregation it was decoupled from. + * Not a fallback for an empty store: it runs on every tick regardless of the store's contents, + * so the path is the normal one rather than one reached only after a failure. The cost is one + * read per tick, in every role that registers it. */ @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.API, process: Process.LATEST_BALANCE_CACHE }) async refreshLatestBalance(): Promise { diff --git a/src/tracing.ts b/src/tracing.ts index 760703c567..3a66ac279e 100644 --- a/src/tracing.ts +++ b/src/tracing.ts @@ -96,13 +96,12 @@ export function isTelemetryEnabled(): boolean { } /** - * The name both processes report to the collector. Without the distinction they would report as - * one service, and a consumer of the traces could not tell the worker's outgoing calls apart from - * the calls a request made. + * Maps CRON_ROLE to the service name reported with every span, so spans from the worker role are + * distinguishable from the rest instead of arriving under one name. * - * Reads the environment directly rather than the configuration: tracing starts before anything - * else, and importing the configuration here would pull in the instrumented modules before the - * SDK has had a chance to patch them. The value is validated where the configuration is built. + * Reads the environment directly rather than the configuration: startTracing runs before the + * application is created, and importing the configuration here would load the instrumented + * modules before the SDK patches them. The value itself is validated in config.ts. */ export function tracingServiceName(): string { return process.env.CRON_ROLE === 'worker' ? 'dfx-api-worker' : 'dfx-api'; From 96b007a48cff28266da74fbdd68e1fd4d98d3878 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:43 +0200 Subject: [PATCH 25/86] Absorb what develop added, and tighten the two new mechanisms develop gained two cron jobs while this branch was open, and both needed a scope. StaffKycClearanceService writes its result to a setting, so it belongs to the worker. ProcessService::resyncStaffKycClearance reads that setting into module-global state a guard evaluates on the request path - `both`, like its siblings, and for a reason worth naming: the allowlist fails closed, so frozen at boot it would lock out staff whose clearance arrives later. From the review round: - The retry around the monitoring write is restricted to the errors a second attempt can resolve - unique violation, deadlock, serialization failure. It repeated everything before, including a malformed row or a permission error, which only puts the same failing statement on the database twice. - An unreadable timestamp now counts as the oldest possible instead of producing NaN, where the comparison silently stopped applying. - The start-up fill logs its failure. Not rethrown - the endpoint already handles an empty store and the scheduled run retries - but a silent failure would leave no trace of why the endpoint is empty. - CONTRIBUTING still showed the enum members under their old casing, so the examples no longer compiled. The inventory is regenerated from the source: 136 jobs, 116 worker, 6 api, 14 both, and the flag section counts the fourth resync job that joined the three deliberate ones. --- CONTRIBUTING.md | 6 +- docs/cron-jobs.md | 30 +++++----- src/shared/services/process.service.ts | 5 +- .../__tests__/monitoring.service.spec.ts | 57 +++++++++++++------ .../core/monitoring/monitoring.service.ts | 24 ++++++-- .../services/payment-cron.service.ts | 3 +- .../user/staff-kyc-clearance.service.ts | 4 +- .../dashboard-financial.service.spec.ts | 55 +++++++++++++++--- .../dashboard/dashboard-financial.service.ts | 16 ++++-- 9 files changed, 145 insertions(+), 55 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 89c84d6eed..a17bfd9c17 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -549,18 +549,18 @@ compiler enforces it: `scope` is a mandatory parameter of `@DfxCron`. ```typescript // Worker: writes to the database, moves money, or calls an external system in a way that // changes state. The normal case. -@DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.Worker, process: Process.PAYMENT }) +@DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.PAYMENT }) async processPayments(): Promise {} // Both: the effect is confined to the process it runs in — refreshing an in-memory copy of // global state, expiring a local cache, measuring this process. It runs everywhere, because // requests on the API process read what it maintains. -@DfxCron(CronExpression.EVERY_30_SECONDS, { scope: CronScope.Both }) +@DfxCron(CronExpression.EVERY_30_SECONDS, { scope: CronScope.BOTH }) async resyncDeniedJwtAccounts(): Promise {} // Api: maintains state read only from a request path, or drives work bound to the connections // this process holds open. -@DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.Api, process: Process.UPDATE_STATISTIC }) +@DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.API, process: Process.UPDATE_STATISTIC }) async doUpdate(): Promise {} ``` diff --git a/docs/cron-jobs.md b/docs/cron-jobs.md index 79ca8b144d..7edee0f5db 100644 --- a/docs/cron-jobs.md +++ b/docs/cron-jobs.md @@ -1,6 +1,6 @@ # Cron jobs -Every scheduled job this service runs: **134 `@DfxCron` declarations** across 94 files and 34 areas. +Every scheduled job this service runs: **136 `@DfxCron` declarations** across 95 files and 34 areas. ## Columns @@ -15,7 +15,7 @@ Every scheduled job this service runs: **134 `@DfxCron` declarations** across 94 ## Scopes `scope` is a mandatory parameter of `@DfxCron` and says which process registers the job: -115 are `worker`, 6 are `api`, 13 are `both`. `CRON_ROLE` decides what a process is +116 are `worker`, 6 are `api`, 14 are `both`. `CRON_ROLE` decides what a process is (`worker`, `api`, or `all` for a single-process setup); a process runs its own scope plus `both`. `worker` is the normal case — anything writing to the database or driving business forward belongs @@ -30,21 +30,23 @@ request path loads on demand, and a job may refresh it but must not be the only ## Flags -116 of the 134 jobs carry a `process` flag, 18 do not. A job with a flag can be switched off +116 of the 136 jobs carry a `process` flag, 20 do not. A job with a flag can be switched off without a deploy — `DfxCronService` skips it when the process appears in the disabled set, which `ProcessService` refreshes from the `disabledProcesses` setting and the `DISABLED_PROCESSES` environment variable every 30 seconds. -A job **without** a flag runs unconditionally. That is deliberate for three of them — the -`ProcessService::resync*` jobs maintain the disabled set and the JWT denylists themselves, so -making them switchable would let a configuration change disable the mechanism that reads -configuration changes. For the remaining 15 it is simply an omission: +A job **without** a flag runs unconditionally. That is deliberate for four of them — the +`ProcessService::resync*` jobs maintain the disabled set, the JWT denylists and the staff +clearance allowlist themselves, so making them switchable would let a configuration change +disable the mechanism that reads configuration changes. For the remaining 16 it is simply an +omission: | Job | Interval | | --- | --- | | `ExchangeController::checkTrades` | 30 seconds | | `AuthService::checkLists` | minute | | `TransactionController::checkLists` | minute | +| `StaffKycClearanceService::syncStaffKycClearance` | minute | | `UserDataService::processCleanupMailSecretCache` | minute | | `TransactionHelper::updateCache` | 5 minutes | | `BuyService` / `SellService` / `SwapService` / `UserService` / `UserDataService` `::resetMonthlyVolumes` | 1st of month | @@ -58,8 +60,8 @@ New jobs should declare a flag unless there is a reason like the one above. | -------- | ---: | | second | 5 | | 10 seconds | 3 | -| 30 seconds | 8 | -| minute | 51 | +| 30 seconds | 9 | +| minute | 52 | | 5 minutes | 17 | | 10 minutes | 15 | | hour | 16 | @@ -77,7 +79,7 @@ Jobs by area: | Area | Jobs | Without flag | | ---- | ---: | -----------: | -| `subdomains/generic/user` | 15 | 6 | +| `subdomains/generic/user` | 16 | 7 | | `subdomains/core/monitoring` | 14 | — | | `subdomains/core/accounting` | 13 | — | | `subdomains/supporting/payin` | 12 | — | @@ -85,13 +87,13 @@ Jobs by area: | `subdomains/core/buy-crypto` | 6 | 4 | | `subdomains/core/sell-crypto` | 5 | 2 | | `subdomains/supporting/payment` | 5 | 1 | +| `shared/services` | 4 | 4 | | `subdomains/core/payment-link` | 4 | — | | `subdomains/generic/kyc` | 4 | — | -| `subdomains/supporting/bank-tx` | 4 | — | | `subdomains/supporting/bank` | 4 | — | +| `subdomains/supporting/bank-tx` | 4 | — | | `subdomains/supporting/fiat-output` | 4 | — | | `subdomains/supporting/support-issue` | 4 | — | -| `shared` | 3 | 3 | | `subdomains/core/liquidity-management` | 3 | — | | `subdomains/core/referral` | 3 | — | | `subdomains/core/trading` | 3 | — | @@ -117,7 +119,7 @@ Jobs by area: Every `@DfxCron(` occurrence in `src/**/*.ts`. Decorator arguments are read by a balanced-paren scan, so multi-line declarations are included — a line-based match misses 26 of them. Interval, flag and scope come from those arguments, so all three are as accurate as the source. The parsed -count is asserted against a raw text count of the decorator: **134 = 134**, no gap. Class and +count is asserted against a raw text count of the decorator: **136 = 136**, no gap. Class and method come from the enclosing `export class` (including `export abstract class`) and the identifier following the decorator. @@ -149,6 +151,7 @@ the interval while running as an independent timer with its own lock. | 30 seconds | — | `both` | `ProcessService::resyncDeniedJwtAccounts` | `shared/services/process.service.ts` | | 30 seconds | — | `both` | `ProcessService::resyncDeniedJwtAddresses` | `shared/services/process.service.ts` | | 30 seconds | — | `both` | `ProcessService::resyncDisabledProcesses` | `shared/services/process.service.ts` | +| 30 seconds | — | `both` | `ProcessService::resyncStaffKycClearance` | `shared/services/process.service.ts` | | minute | `PAY_OUT` | `worker` | `AdminService::completeLiquidityOrders` | `subdomains/generic/admin/admin.service.ts` | | minute | `MONITORING` | `worker` | `AmlObserver::fetch` | `subdomains/core/monitoring/observers/aml.observer.ts` | | minute | — | `both` | `AuthService::checkLists` | `subdomains/generic/user/models/auth/auth.service.ts` | @@ -190,6 +193,7 @@ the interval while running as an independent timer with its own lock. | minute | `PAYMENT_EXPIRATION` | `api` | `PaymentCronService::processExpiredPayments` | `subdomains/core/payment-link/services/payment-cron.service.ts` | | minute | `UPDATE_BLOCKCHAIN_FEE` | `both` | `PaymentLinkFeeService::updateFees` | `subdomains/core/payment-link/services/payment-link-fee.service.ts` | | minute | `REALUNIT_QUOTE_COMPLETION` | `worker` | `RealUnitJobService::completeSettledQuotes` | `subdomains/supporting/realunit/realunit-job.service.ts` | +| minute | — | `worker` | `StaffKycClearanceService::syncStaffKycClearance` | `subdomains/generic/user/models/user/staff-kyc-clearance.service.ts` | | minute | `SUPPORT_BOT` | `worker` | `SupportIssueJobService::sendAutoResponses` | `subdomains/supporting/support-issue/services/support-issue-job.service.ts` | | minute | `TFA_CACHE` | `both` | `TfaService::processCleanupSecretCache` | `subdomains/generic/kyc/services/tfa.service.ts` | | minute | `TRADING` | `worker` | `TradingJobService::processOrders` | `subdomains/core/trading/services/trading-job.service.ts` | diff --git a/src/shared/services/process.service.ts b/src/shared/services/process.service.ts index bbbcbd78d8..da65245452 100644 --- a/src/shared/services/process.service.ts +++ b/src/shared/services/process.service.ts @@ -207,7 +207,10 @@ export class ProcessService implements OnModuleInit { } // Primes the fail-closed staff clearance allowlist — see `staff-kyc-clearance.ts` for the semantics. - @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800 }) + // Both, like the sibling resync jobs above: the allowlist it fills is module-global state that a + // guard reads on the request path, so it has to be refreshed in the process serving those + // requests. Frozen at boot it fails closed, locking out staff whose clearance arrives later. + @DfxCron(CronExpression.EVERY_30_SECONDS, { timeout: 1800, scope: CronScope.BOTH }) async resyncStaffKycClearance(): Promise { const list = await this.settingService.getStaffKycClearance(); SetStaffKycClearance(list); diff --git a/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts b/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts index 0d39a06f9e..4b48e5e288 100644 --- a/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts +++ b/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts @@ -115,6 +115,31 @@ describe('MonitoringService', () => { }); }); + /** Lets the first transaction fail with the given database error, then behaves normally. */ + function failFirstWith(error: { code: string; message: string }): () => number { + let attempts = 0; + + Object.defineProperty(repo, 'manager', { + value: { + transaction: (run: (m: unknown) => Promise) => { + attempts++; + if (attempts === 1) return Promise.reject(Object.assign(new Error(error.message), { code: error.code })); + + return run({ + findOne: jest.fn().mockResolvedValue(snapshot(persisted)), + save: jest.fn().mockImplementation((_e: unknown, row: { id: number; data: string }) => { + written.push(row); + return Promise.resolve(row); + }), + }); + }, + }, + configurable: true, + }); + + return () => attempts; + } + describe('persisting the state', () => { it('keeps metrics another process wrote', async () => { // Every process subscribes to its own updates and writes the same single row. Replacing it @@ -169,33 +194,29 @@ describe('MonitoringService', () => { // An absent row cannot be locked, so two writers can reach the insert together and one // loses on the primary key. Without the retry that process's change is dropped until the // metric happens to change again. - let attempts = 0; - Object.defineProperty(repo, 'manager', { - value: { - transaction: (run: (m: unknown) => Promise) => { - attempts++; - if (attempts === 1) return Promise.reject(new Error('duplicate key value violates unique constraint')); - return run({ - findOne: jest.fn().mockResolvedValue(snapshot(persisted)), - save: jest.fn().mockImplementation((_e: unknown, row: { id: number; data: string }) => { - written.push(row); - return Promise.resolve(row); - }), - }); - }, - }, - configurable: true, - }); + const attempts = failFirstWith({ code: '23505', message: 'duplicate key value' }); const next: SystemState = { ledger: { open: metric({ count: 1 }, '2020-01-01T00:20:00Z') } }; await service['persist']({}, next); - expect(attempts).toBe(2); + expect(attempts()).toBe(2); expect(JSON.parse(written[0].data).ledger.open.data).toEqual({ count: 1 }); expect(notificationService.sendMail).not.toHaveBeenCalled(); }); + it('does not retry an error a second attempt cannot resolve', async () => { + // The retry exists for the insert conflict and for deadlocks. Repeating a malformed row or + // a permission error would only put the same failing statement on the database twice. + const attempts = failFirstWith({ code: '42501', message: 'permission denied' }); + + await service['persist']({}, { ledger: { open: metric({ count: 1 }, '2020-01-01T00:20:00Z') } }); + + expect(attempts()).toBe(1); + expect(written).toEqual([]); + expect(notificationService.sendMail).toHaveBeenCalled(); + }); + it('writes a metric that did not exist before', async () => { const next: SystemState = { ledger: { open: metric({ count: 3 }, '2020-01-01T00:20:00Z') } }; diff --git a/src/subdomains/core/monitoring/monitoring.service.ts b/src/subdomains/core/monitoring/monitoring.service.ts index 727bb2c093..81bdcdc6fe 100644 --- a/src/subdomains/core/monitoring/monitoring.service.ts +++ b/src/subdomains/core/monitoring/monitoring.service.ts @@ -22,6 +22,9 @@ type SubsystemObservers = Map>; export class MonitoringService implements OnModuleInit { private static readonly stateCacheMs = 30 * 1000; + /** Postgres: unique violation, deadlock, serialization failure - all resolvable by trying again. */ + private static readonly retryableWriteCodes = ['23505', '40P01', '40001']; + private readonly logger = new DfxLogger(MonitoringService); #$state: BehaviorSubject = new BehaviorSubject({}); @@ -153,8 +156,16 @@ export class MonitoringService implements OnModuleInit { // A second attempt covers the case where the row does not exist yet: it cannot be locked, // so two writers can reach the insert together and one loses on the primary key. Retrying // finds the row the other one created and merges into it, instead of dropping this - // process's change until the metric happens to change again. - const merged = await Util.retry(() => this.mergeIntoStoredState(changed, newState), 2); + // process's change until the metric happens to change again. Restricted to the errors that + // a retry can actually resolve - a unique violation, a deadlock or a serialization failure. + // Anything else fails once and is reported. + const merged = await Util.retry( + () => this.mergeIntoStoredState(changed, newState), + 2, + 0, + undefined, + (e) => MonitoringService.retryableWriteCodes.includes((e as { code?: string })?.code), + ); this.#storedState = { state: merged, loaded: Date.now() }; } catch (e) { @@ -272,9 +283,14 @@ export class MonitoringService implements OnModuleInit { return merged; } - /** Parsed JSON carries `updated` as a string, the in-memory state as a Date. */ + /** + * Parsed JSON carries `updated` as a string, the in-memory state as a Date. A missing or + * unparsable value counts as the oldest possible, so a metric carrying one never wins a + * comparison against a readable timestamp - and never blocks one either. + */ private updatedAt(metric?: Metric): number { - return metric?.updated ? new Date(metric.updated).getTime() : 0; + const time = metric?.updated ? new Date(metric.updated).getTime() : 0; + return Number.isFinite(time) ? time : 0; } private getSubsystemState(state: SystemState, subsystem: string): SubsystemState { diff --git a/src/subdomains/core/payment-link/services/payment-cron.service.ts b/src/subdomains/core/payment-link/services/payment-cron.service.ts index 7e3f593822..0545752afe 100644 --- a/src/subdomains/core/payment-link/services/payment-cron.service.ts +++ b/src/subdomains/core/payment-link/services/payment-cron.service.ts @@ -22,7 +22,8 @@ export class PaymentCronService { // subscribes to. Both hold their state in the instance, so they only reach a caller of this same // process. `Both` is no option either: the jobs write to the database and trigger merchant // webhooks, which a second registration would repeat - the lock in DfxCronService is per - // process. + // process. That holds for exactly one process per role: the lock cannot span processes, so a + // second instance of the same role would double these writes just as `Both` would. @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.API, process: Process.PAYMENT_EXPIRATION }) async processExpiredPayments(): Promise { await this.paymentLinkPaymentService.processExpiredPayments(); diff --git a/src/subdomains/generic/user/models/user/staff-kyc-clearance.service.ts b/src/subdomains/generic/user/models/user/staff-kyc-clearance.service.ts index 6aec42ed42..57b55d8442 100644 --- a/src/subdomains/generic/user/models/user/staff-kyc-clearance.service.ts +++ b/src/subdomains/generic/user/models/user/staff-kyc-clearance.service.ts @@ -3,7 +3,7 @@ import { CronExpression } from '@nestjs/schedule'; import { rolesSatisfying } from 'src/shared/auth/role.guard'; import { KycGatedRoles } from 'src/shared/auth/user-role.enum'; import { SettingService } from 'src/shared/models/setting/setting.service'; -import { DfxCron } from 'src/shared/utils/cron'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { In, Raw } from 'typeorm'; import { UserRepository } from './user.repository'; @@ -51,7 +51,7 @@ export class StaffKycClearanceService { // Every minute, matching JwtRevocationSyncService: revoking elevated access promptly is a security // requirement and warrants the same exception to the "prefer 15min" cron guideline. - @DfxCron(CronExpression.EVERY_MINUTE, { timeout: 1800 }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, timeout: 1800 }) async syncStaffKycClearance(): Promise { const staffUsers = await this.userRepo.find({ select: { id: true, userData: { id: true } }, diff --git a/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts b/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts index 5d4a725b6b..ef7063bf85 100644 --- a/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts +++ b/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts @@ -10,6 +10,7 @@ import { FinancialLogSummary } from '../../log/log.repository'; import { LogService } from '../../log/log.service'; import { DashboardFinancialService } from '../dashboard-financial.service'; import { LatestBalanceResponseDto } from '../dto/financial-log.dto'; +import { Config, CronRole } from 'src/config/config'; import { TestUtil } from 'src/shared/utils/test.util'; import { LatestBalanceStore } from '../latest-balance.store'; @@ -19,6 +20,14 @@ describe('DashboardFinancialService', () => { let assetService: AssetService; let latestBalanceStore: LatestBalanceStore; + // Config is only populated once TestUtil.provideConfig has built it, so the original role is + // captured after the module is compiled, not while the suite is being defined. + let originalRole: CronRole; + + afterEach(() => { + if (originalRole !== undefined) Config.cronRole = originalRole; + }); + beforeEach(async () => { logService = createMock(); assetService = createMock(); @@ -36,8 +45,17 @@ describe('DashboardFinancialService', () => { }).compile(); service = module.get(DashboardFinancialService); + originalRole ??= Config.cronRole; }); + function logEntry(): Log { + return { + id: 1, + created: new Date('2026-07-14T12:00:00Z'), + message: JSON.stringify({ assets: {}, balancesByFinancialType: {} }), + } as Log; + } + /** Mocks the log entry and the assets the job resolves, then runs it. */ async function refreshFrom( timestamp: Date, @@ -502,14 +520,11 @@ describe('DashboardFinancialService', () => { expect(latestBalanceStore.set).toHaveBeenCalledWith(expected); }); - it('fills the store at start-up instead of waiting for the first scheduled run', async () => { - // The endpoint answers from the store alone. Without this the window after a restart is a - // full cron interval wide, and it does not shrink by waiting. - jest.spyOn(logService, 'getLatestFinancialLog').mockResolvedValue({ - id: 1, - created: new Date('2026-07-14T12:00:00Z'), - message: JSON.stringify({ assets: {}, balancesByFinancialType: {} }), - } as Log); + it.each([CronRole.ALL, CronRole.API])('fills the store at start-up in the %s role', async (role) => { + // getLatestBalance answers from the store and nothing else, so until the first fill the + // endpoint has no value to return. + Config.cronRole = role; + jest.spyOn(logService, 'getLatestFinancialLog').mockResolvedValue(logEntry()); service.onModuleInit(); await new Promise(process.nextTick); @@ -517,6 +532,30 @@ describe('DashboardFinancialService', () => { expect(latestBalanceStore.set).toHaveBeenCalled(); }); + it('does not fill it in the worker role, where nothing reads the store', async () => { + Config.cronRole = CronRole.WORKER; + const getLatestFinancialLogSpy = jest.spyOn(logService, 'getLatestFinancialLog'); + + service.onModuleInit(); + await new Promise(process.nextTick); + + expect(getLatestFinancialLogSpy).not.toHaveBeenCalled(); + expect(latestBalanceStore.set).not.toHaveBeenCalled(); + }); + + it('logs the failure of the start-up fill instead of swallowing it', async () => { + // The scheduled run retries a minute later, so this must not throw - but a silent failure + // would leave the endpoint empty with no trace of why. + Config.cronRole = CronRole.API; + jest.spyOn(logService, 'getLatestFinancialLog').mockRejectedValue(new Error('database unavailable')); + const errorSpy = jest.spyOn(service['logger'], 'error').mockImplementation(() => undefined); + + service.onModuleInit(); + await new Promise(process.nextTick); + + expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('start-up'), expect.any(Error)); + }); + it('leaves the store untouched when there is no log entry yet', async () => { jest.spyOn(logService, 'getLatestFinancialLog').mockResolvedValue(undefined); diff --git a/src/subdomains/supporting/dashboard/dashboard-financial.service.ts b/src/subdomains/supporting/dashboard/dashboard-financial.service.ts index b9016e745d..cf1b74b91d 100644 --- a/src/subdomains/supporting/dashboard/dashboard-financial.service.ts +++ b/src/subdomains/supporting/dashboard/dashboard-financial.service.ts @@ -34,11 +34,17 @@ export class DashboardFinancialService implements OnModuleInit { ) {} onModuleInit() { - // Fills the store once at start-up instead of leaving it empty until the first scheduled run. - // The endpoint answers from the store alone, so without this the window after a restart is a - // full cron interval wide - and it does not shrink by waiting, because the job that used to - // fill the store on the spot is gone. - if (Config.cronRole !== CronRole.WORKER) void this.refreshLatestBalance().catch(() => undefined); + // Fills the store once at start-up instead of leaving it empty until the first scheduled run: + // getLatestBalance answers from the store and nothing else, so until the first fill the + // endpoint has no value to return. + if (Config.cronRole === CronRole.WORKER) return; + + void this.refreshLatestBalance().catch((e) => + // Not rethrown: a failed first fill leaves the store empty, which the endpoint already + // handles, and the scheduled run retries a minute later. Swallowing it silently would hide + // why the endpoint is empty in the meantime. + this.logger.error('Failed to fill the latest balance store at start-up:', e), + ); } async getFinancialLog(from?: Date, dailySample?: boolean, includeByType?: boolean): Promise { From 466a9888f27b604d8a5c5a3861de675f5c155141 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:45 +0200 Subject: [PATCH 26/86] Make the contribution guide compile against its own rule The examples still called @DfxCron without the scope the same document now declares mandatory, so a reader copying them would get a type error. The anti-pattern table pointed at the same incomplete call. The payment job comment also claimed how many processes run per role. That is a property of a deployment, not of this repository - what holds here is that the lock lives in the process, so it cannot stop a second one from running the same job. That is the reason `Both` is excluded, and it is the reason regardless of how many processes there are. --- CONTRIBUTING.md | 8 ++++---- src/subdomains/core/monitoring/monitoring.service.ts | 5 ++++- .../core/payment-link/services/payment-cron.service.ts | 5 ++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a17bfd9c17..768f5b167c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -517,7 +517,7 @@ Use `@DfxCron` (custom wrapper with built-in locking, process control, and error ```typescript // GOOD: @DfxCron handles everything -@DfxCron(CronExpression.EVERY_MINUTE, { process: Process.PAYMENT, timeout: 1800 }) +@DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.PAYMENT, timeout: 1800 }) async processPayments(): Promise { // no @Lock, no DisabledProcess check needed } @@ -871,13 +871,13 @@ const isValid = await this.validateIban(iban).catch(() => false); ```typescript // BAD: @DfxCron already handles errors -@DfxCron(CronExpression.EVERY_HOUR) +@DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.WORKER }) async process(): Promise { try { ... } catch (e) { this.logger.error(e); } // redundant } // GOOD -@DfxCron(CronExpression.EVERY_HOUR) +@DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.WORKER }) async process(): Promise { // just do the work } @@ -1281,7 +1281,7 @@ single DTO with two fields (PR #3772, 91 LOC, ~50% reduction). | Loading all then filtering in JS | SQL WHERE clause | | `any` type | Proper typed interface/class | | `string` for enum values | Typed enum | -| `@Interval(60000)` | `@DfxCron(CronExpression.EVERY_MINUTE)` | +| `@Interval(60000)` | `@DfxCron(..., { scope })` — see Cron Jobs | | `eager: true` everywhere | Explicit relation loading | | Providing service in multiple modules | Single module, import from there | | `JSON.stringify(JSON.parse(...))` | Unnecessary — remove | diff --git a/src/subdomains/core/monitoring/monitoring.service.ts b/src/subdomains/core/monitoring/monitoring.service.ts index 81bdcdc6fe..4c13a662ce 100644 --- a/src/subdomains/core/monitoring/monitoring.service.ts +++ b/src/subdomains/core/monitoring/monitoring.service.ts @@ -22,7 +22,10 @@ type SubsystemObservers = Map>; export class MonitoringService implements OnModuleInit { private static readonly stateCacheMs = 30 * 1000; - /** Postgres: unique violation, deadlock, serialization failure - all resolvable by trying again. */ + /** + * Postgres: unique violation, deadlock, serialization failure - all resolvable by trying again. + * Read from `code` on the error, the same way LedgerAccountService reads a unique violation. + */ private static readonly retryableWriteCodes = ['23505', '40P01', '40001']; private readonly logger = new DfxLogger(MonitoringService); diff --git a/src/subdomains/core/payment-link/services/payment-cron.service.ts b/src/subdomains/core/payment-link/services/payment-cron.service.ts index 0545752afe..69337009db 100644 --- a/src/subdomains/core/payment-link/services/payment-cron.service.ts +++ b/src/subdomains/core/payment-link/services/payment-cron.service.ts @@ -21,9 +21,8 @@ export class PaymentCronService { // waitForPayment awaits and pushes the device activation into the RxJS subject PaymentLinkGateway // subscribes to. Both hold their state in the instance, so they only reach a caller of this same // process. `Both` is no option either: the jobs write to the database and trigger merchant - // webhooks, which a second registration would repeat - the lock in DfxCronService is per - // process. That holds for exactly one process per role: the lock cannot span processes, so a - // second instance of the same role would double these writes just as `Both` would. + // webhooks, which a second registration would repeat - the lock DfxCronService creates lives in + // the process, so it cannot prevent a second process from running the same job. @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.API, process: Process.PAYMENT_EXPIRATION }) async processExpiredPayments(): Promise { await this.paymentLinkPaymentService.processExpiredPayments(); From 33d24fc1f34e9338d877a2d88819e6489e9061ee Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:46 +0200 Subject: [PATCH 27/86] Say what the boot count actually counts, and widen the guard by two decorators The log line claimed the discovery sees jobs on dynamically resolved providers. It does not: providers whose dependency tree is not static are filtered out before the scan, so a job declared there is registered in no role at all. What the count is good for stands - it is what this process registered, which differs from counting decorators in the source because an abstract base class is registered once per concrete provider - but the reason given for it was wrong. The guard now also rejects @Interval and @Timeout. They register with the same scheduler and are just as invisible to the scope as the native @Cron it already covered. And it states where it stops: a timer built from a repeating setTimeout, an aliased import or a scheduler reached through an object property passes unseen. A text match cannot see those. Writing that down is worth more than implying a completeness the check does not have. --- CONTRIBUTING.md | 2 +- .../services/__tests__/cron-registration.guard.spec.ts | 10 ++++++++++ src/shared/services/dfx-cron.service.ts | 8 ++++---- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 768f5b167c..1ec036a862 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1281,7 +1281,7 @@ single DTO with two fields (PR #3772, 91 LOC, ~50% reduction). | Loading all then filtering in JS | SQL WHERE clause | | `any` type | Proper typed interface/class | | `string` for enum values | Typed enum | -| `@Interval(60000)` | `@DfxCron(..., { scope })` — see Cron Jobs | +| `@Interval(60000)` | `@DfxCron(EVERY_MINUTE, { scope: CronScope.WORKER })` | | `eager: true` everywhere | Explicit relation loading | | Providing service in multiple modules | Single module, import from there | | `JSON.stringify(JSON.parse(...))` | Unnecessary — remove | diff --git a/src/shared/services/__tests__/cron-registration.guard.spec.ts b/src/shared/services/__tests__/cron-registration.guard.spec.ts index c881ec9bcc..5f8b4f430b 100644 --- a/src/shared/services/__tests__/cron-registration.guard.spec.ts +++ b/src/shared/services/__tests__/cron-registration.guard.spec.ts @@ -11,6 +11,11 @@ const SRC = join(__dirname, '..', '..', '..'); * The check is syntactic on purpose. It asks whether a pattern occurs, not whether the code * behind it is safe, so its exception list has a natural ceiling and every entry is a deliberate * decision rather than a special case. + * + * Its reach ends there, and that is worth stating: a timer built from a repeating setTimeout, an + * aliased import or a scheduler reached through an object property passes unseen. Catching those + * needs an AST-based rule rather than a text match. What this covers is the shape the four cases + * in this repository actually had. */ const FORBIDDEN: { pattern: RegExp; what: string; instead: string }[] = [ { @@ -18,6 +23,11 @@ const FORBIDDEN: { pattern: RegExp; what: string; instead: string }[] = [ what: 'the native @Cron decorator', instead: 'use @DfxCron, which applies the scope, the process flag and the lock', }, + { + pattern: /@Interval\(|@Timeout\(/, + what: 'the @Interval or @Timeout decorator', + instead: 'use @DfxCron - these register with the same scheduler and are equally invisible to the scope', + }, { pattern: /\bsetInterval\(/, what: 'a bare setInterval', diff --git a/src/shared/services/dfx-cron.service.ts b/src/shared/services/dfx-cron.service.ts index 6de480ca0f..a0d399402a 100644 --- a/src/shared/services/dfx-cron.service.ts +++ b/src/shared/services/dfx-cron.service.ts @@ -60,10 +60,10 @@ export class DfxCronService implements OnModuleInit { }); }); - // Counts what this process actually registered, including jobs reaching the scheduler - // through a dynamically resolved provider or an abstract base class - the discovery above - // sees those, a list of decorators would not. Stays on `info` so the split is readable - // without changing the log level. + // Counts what this process actually registered, which is not the same as counting decorators + // in the source: a job declared on an abstract base class is registered once per concrete + // provider, and the filter above skips providers whose dependency tree is not static. Stays + // on `info` so the split is readable without changing the log level. const total = registered.length + skipped; const byScope = Object.values(CronScope) .map((scope) => `${scope}: ${registered.filter((s) => s === scope).length}`) From a0671931cbb0c96c55f7b234179c3488b8807a0e Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:47 +0200 Subject: [PATCH 28/86] Qualify the enum in the last documentation example The anti-pattern table showed a bare EVERY_MINUTE. Every other example in the file qualifies it, and unqualified it does not resolve. --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1ec036a862..a9984e80e1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1281,7 +1281,7 @@ single DTO with two fields (PR #3772, 91 LOC, ~50% reduction). | Loading all then filtering in JS | SQL WHERE clause | | `any` type | Proper typed interface/class | | `string` for enum values | Typed enum | -| `@Interval(60000)` | `@DfxCron(EVERY_MINUTE, { scope: CronScope.WORKER })` | +| `@Interval(60000)` | `@DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER })` | | `eager: true` everywhere | Explicit relation loading | | Providing service in multiple modules | Single module, import from there | | `JSON.stringify(JSON.parse(...))` | Unnecessary — remove | From 448fcd8205eb97159419f45ed5d701ad9f6a4319 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:49 +0200 Subject: [PATCH 29/86] Close the two gaps a CONTRIBUTING audit turned up The guide contradicted itself on bare @Cron. The opening example still offered it as the alternative you may pick when you handle @Lock and DisabledProcess yourself, while the section this PR adds says a test rejects it. Both stood in the same file. A bare @Cron carries no scope and therefore registers in every process, so the guard is the rule and the example is only there to show what the wrapper takes off your hands. Says so now, and points at the section. The new refreshLatestBalance runs every minute where the guide asks for fifteen. The reason was never written down: LogJobService writes the entry it reads at exactly that interval (TRADING_LOG, EVERY_MINUTE). A longer interval here saves no write, it only serves a staler value than the data allows. --- CONTRIBUTING.md | 4 +++- .../supporting/dashboard/dashboard-financial.service.ts | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a9984e80e1..fa5775e6ee 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -522,7 +522,9 @@ async processPayments(): Promise { // no @Lock, no DisabledProcess check needed } -// ONLY with bare @Cron: manual @Lock + DisabledProcess required +// What the wrapper takes off your hands — NOT an alternative you may pick. +// A bare @Cron carries no scope, so it registers in every process; a guard test +// rejects it (see "Register periodic work through @DfxCron" below). @Cron(CronExpression.EVERY_MINUTE) @Lock(1800) async processPayments(): Promise { diff --git a/src/subdomains/supporting/dashboard/dashboard-financial.service.ts b/src/subdomains/supporting/dashboard/dashboard-financial.service.ts index cf1b74b91d..dd8787f55b 100644 --- a/src/subdomains/supporting/dashboard/dashboard-financial.service.ts +++ b/src/subdomains/supporting/dashboard/dashboard-financial.service.ts @@ -158,6 +158,10 @@ export class DashboardFinancialService implements OnModuleInit { * Not a fallback for an empty store: it runs on every tick regardless of the store's contents, * so the path is the normal one rather than one reached only after a failure. The cost is one * read per tick, in every role that registers it. + * + * Every minute rather than the 15 minutes CONTRIBUTING prefers, because that is the interval + * LogJobService already writes the underlying entry at (TRADING_LOG, EVERY_MINUTE). A longer + * one here would not save a write, it would only serve a staler value than the data allows. */ @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.API, process: Process.LATEST_BALANCE_CACHE }) async refreshLatestBalance(): Promise { From 5c1262ff77a7b83c701b63901788515155aac651 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:50 +0200 Subject: [PATCH 30/86] Fix what a PR-level review found outside the diff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four defects, none of which a diff review could see, plus the numbers they move. The monitoring read path took the highest id while the write path always targets id 1. With a second row present — and the documented prod drift on this table lists exactly that — every read would have returned a row that is never written, so /health and /monitoring/data would have served a frozen state without an error. Both sides now name the same row. updateFees was scoped Both and queries gas prices for eight EVM chains plus Bitcoin and Firo fee estimates every minute. Its cache has exactly one reader, getMinFee, and following that chain out lands only on request paths and the Api-scoped payment crons; the single Worker job in that domain takes its fee rate from BitcoinFeeService. In the worker those calls would spend quota for a value nothing there reads — and the rule this PR itself adds rules out paid external calls for Both. Now Api, with the chain written down. .env.local.example carried no CRON_ROLE. It is the template the documented local setup copies, so `npm run setup` would have died on a boot that rejects the missing value — the same file spells out the other fail-loud variable, so leaving this one out was an omission, not a decision. The metric export interval had no entry in .env.example although tracing.ts reads and validates it. The discard branch in the snapshot merge now logs. The comment above it says the value does not come back on its own; a run of these lines is the signal that two writers compete for one metric, which today should not happen. docs/cron-jobs.md regenerated from the source: 136 jobs, now 116 worker, 7 api, 13 both. The worker process therefore registers 129 of 136, the api process 20 of 136 — the api-side count is unchanged because the job moved between the two scopes that process registers. --- .env.example | 4 ++++ .env.local.example | 5 +++++ docs/cron-jobs.md | 4 ++-- .../core/monitoring/monitoring.service.ts | 12 ++++++++++-- .../services/payment-link-fee.service.ts | 15 ++++++++++++++- 5 files changed, 35 insertions(+), 5 deletions(-) diff --git a/.env.example b/.env.example index 1cffa3cd38..dcc083f0cc 100644 --- a/.env.example +++ b/.env.example @@ -292,6 +292,10 @@ S3_ADMIN_SECRET_KEY= # OpenTelemetry trace export (OTLP/HTTP). Point this at an OTLP collector to # enable distributed tracing; leave empty to disable. e.g. http://localhost:4318 OTEL_EXPORTER_OTLP_ENDPOINT= +# Metric export interval in milliseconds. Optional: unset leaves the SDK default (60s). +# A shorter interval costs a full collect-and-export of every instrument on the event loop this +# split is meant to keep free. An invalid or non-positive value aborts the boot. +# OTEL_METRIC_EXPORT_INTERVAL=60000 FIXER_BASE_URL= FIXER_API_KEY= diff --git a/.env.local.example b/.env.local.example index c81eb259e0..9e7143ff11 100644 --- a/.env.local.example +++ b/.env.local.example @@ -61,3 +61,8 @@ MAIL_PASS=dummy-password-for-local-dev # compared while REALUNIT_W2W_GAS_WALLET_PRIVATE_KEY/_ADDRESS are unset, so the # value only has to satisfy the boot check. REALUNIT_W2W_GAS_LOW_BALANCE_THRESHOLD=0.05 + +# Which half of the application this process runs: 'all' registers every cron job, which is +# what a local single-process setup wants. There is no default — config.ts rejects a missing or +# unknown value and aborts the boot, so this line is required for `npm run setup` to work. +CRON_ROLE=all diff --git a/docs/cron-jobs.md b/docs/cron-jobs.md index 7edee0f5db..79b7dcd782 100644 --- a/docs/cron-jobs.md +++ b/docs/cron-jobs.md @@ -15,7 +15,7 @@ Every scheduled job this service runs: **136 `@DfxCron` declarations** across 95 ## Scopes `scope` is a mandatory parameter of `@DfxCron` and says which process registers the job: -116 are `worker`, 6 are `api`, 14 are `both`. `CRON_ROLE` decides what a process is +116 are `worker`, 7 are `api`, 13 are `both`. `CRON_ROLE` decides what a process is (`worker`, `api`, or `all` for a single-process setup); a process runs its own scope plus `both`. `worker` is the normal case — anything writing to the database or driving business forward belongs @@ -191,7 +191,7 @@ the interval while running as an independent timer with its own lock. | minute | `PAY_IN` | `worker` | `PayInService::returnPayInEntries` | `subdomains/supporting/payin/services/payin.service.ts` | | minute | `PAYMENT_CONFIRMATIONS` | `api` | `PaymentCronService::checkTxConfirmations` | `subdomains/core/payment-link/services/payment-cron.service.ts` | | minute | `PAYMENT_EXPIRATION` | `api` | `PaymentCronService::processExpiredPayments` | `subdomains/core/payment-link/services/payment-cron.service.ts` | -| minute | `UPDATE_BLOCKCHAIN_FEE` | `both` | `PaymentLinkFeeService::updateFees` | `subdomains/core/payment-link/services/payment-link-fee.service.ts` | +| minute | `UPDATE_BLOCKCHAIN_FEE` | `api` | `PaymentLinkFeeService::updateFees` | `subdomains/core/payment-link/services/payment-link-fee.service.ts` | | minute | `REALUNIT_QUOTE_COMPLETION` | `worker` | `RealUnitJobService::completeSettledQuotes` | `subdomains/supporting/realunit/realunit-job.service.ts` | | minute | — | `worker` | `StaffKycClearanceService::syncStaffKycClearance` | `subdomains/generic/user/models/user/staff-kyc-clearance.service.ts` | | minute | `SUPPORT_BOT` | `worker` | `SupportIssueJobService::sendAutoResponses` | `subdomains/supporting/support-issue/services/support-issue-job.service.ts` | diff --git a/src/subdomains/core/monitoring/monitoring.service.ts b/src/subdomains/core/monitoring/monitoring.service.ts index 4c13a662ce..eeda07b857 100644 --- a/src/subdomains/core/monitoring/monitoring.service.ts +++ b/src/subdomains/core/monitoring/monitoring.service.ts @@ -199,7 +199,13 @@ export class MonitoringService implements OnModuleInit { // the lock while another process wrote a later measurement of the same metric. Writing it // anyway would put the older value back, and it would stay there until the metric changes // again in this process. - if (this.updatedAt(candidate) < this.updatedAt(state[subsystem]?.[metric])) continue; + if (this.updatedAt(candidate) < this.updatedAt(state[subsystem]?.[metric])) { + // Logged rather than dropped silently: the comment above says this value does not come + // back on its own, so a run of these lines is the signal that two writers are competing + // for one metric — which today should not happen, every metric has exactly one writer. + this.logger.info(`Discarding older ${subsystem}/${metric} state while merging the snapshot`); + continue; + } state[subsystem] = { ...(state[subsystem] ?? {}), [metric]: candidate }; } @@ -224,7 +230,9 @@ export class MonitoringService implements OnModuleInit { /** Reads the persisted state without notifying: unlike loadState, this runs on every read. */ private async readState(): Promise { - const latestPersistedState = await this.systemStateSnapshotRepo.findOne({ where: {}, order: { id: 'DESC' } }); + // Reads the row the write path targets. The previous `order: { id: 'DESC' }` took the highest + // id instead, so a second row would have made this read one that is never written. + const latestPersistedState = await this.systemStateSnapshotRepo.findOne({ where: { id: 1 } }); if (!latestPersistedState) { this.logger.warn('No monitoring state found in the database'); diff --git a/src/subdomains/core/payment-link/services/payment-link-fee.service.ts b/src/subdomains/core/payment-link/services/payment-link-fee.service.ts index f2e495331a..91a43c9e40 100644 --- a/src/subdomains/core/payment-link/services/payment-link-fee.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-fee.service.ts @@ -38,7 +38,20 @@ export class PaymentLinkFeeService implements OnModuleInit { } // --- JOBS --- // - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.BOTH, process: Process.UPDATE_BLOCKCHAIN_FEE }) + /** + * Scope Api, not Both: the cache it fills is a field of this service, and the only reader is + * getMinFee below. Following that chain out — createTransferAmount -> createTransferAmounts -> + * createQuote / createPayRequest, plus the Binance webhook handler — every caller is a request + * path or one of the Api-scoped payment crons. The one Worker job in this domain, + * PaymentCronService::forwardDeposits, takes its fee rate from BitcoinFeeService and never + * reaches this cache. + * + * Both would also break the rule this scope mechanism introduced: a job that runs in every + * process must be harmless twice over, and this one queries gas prices for eight EVM chains + * plus Bitcoin and Firo fee estimates on every tick. In the worker those calls would spend + * quota to produce a value nothing there reads. + */ + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.API, process: Process.UPDATE_BLOCKCHAIN_FEE }) async updateFees(): Promise { if (GetConfig().environment === Environment.LOC) return; From 77fef21671ebefbac835243e88876db28e14c135 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:52 +0200 Subject: [PATCH 31/86] feat(cron): report the active role continuously so the alert can read it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Der Boot-Log sagt, mit welcher Rolle ein Prozess gestartet ist. Fuer die Frage, ob beide Prozesse GERADE die richtigen Rollen fahren, taugt er nicht: er entsteht einmal. Ein Alarm ueber einem Zaehlfenster darauf meldet nach Ablauf des Fensters entweder nichts mehr oder dauerhaft etwas — und den gefaehrlichen Fall, dass ein Prozess ohne Neustart mit der falschen Rolle weiterlaeuft, sieht er ueberhaupt nicht. reportRole beantwortet dieselbe Frage laufend. Scope 'both', damit die Zeile in JEDEM Prozess entsteht, und die Rolle steht in der Zeile, sodass eine vertauschte Zuordnung als falsche Rolle sichtbar wird und nicht nur als fehlende Zeile. Kein process-Flag: ein abschaltbarer Waechter sieht abgeschaltet genau so aus wie der Ausfall, den er melden soll. Drei Tests halten fest, was der Alarm voraussetzt und was am Aufrufort nicht sichtbar ist: Scope 'both', kein Flag, Rolle im Text. Ausserdem in docs/cron-jobs.md festgehalten, was beim Nachzaehlen auffiel: ExchangeController::checkTrades wird nie registriert, weil die Klasse nur unter controllers: steht und getProviders() keine Controller liefert. Das gilt unabhaengig von der Prozesstrennung und wird hier nur benannt, nicht repariert. --- CONTRIBUTING.md | 6 ++- docs/cron-jobs.md | 37 ++++++++++++++----- .../__tests__/dfx-cron.service.spec.ts | 37 +++++++++++++++++++ src/shared/services/dfx-cron.service.ts | 24 +++++++++++- 4 files changed, 91 insertions(+), 13 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fa5775e6ee..9152159cfe 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -533,8 +533,10 @@ async processPayments(): Promise { } ``` -Declare a `process` flag unless the job maintains the disabled set itself. Without one the job -runs unconditionally and cannot be switched off without a deploy. +Declare a `process` flag unless the job maintains the disabled set itself, or unless something +outside the process infers health from the job still running — a watchdog that can be switched off +looks, once it is off, exactly like the failure it watches for. Without a flag the job runs +unconditionally and cannot be switched off without a deploy. [docs/cron-jobs.md](docs/cron-jobs.md) lists every scheduled job with its interval, flag and scope. **Adding, removing or re-scheduling a job must be reflected there in the same PR.** diff --git a/docs/cron-jobs.md b/docs/cron-jobs.md index 79b7dcd782..05fed38bd9 100644 --- a/docs/cron-jobs.md +++ b/docs/cron-jobs.md @@ -1,6 +1,6 @@ # Cron jobs -Every scheduled job this service runs: **136 `@DfxCron` declarations** across 95 files and 34 areas. +Every scheduled job this service runs: **137 `@DfxCron` declarations** across 96 files and 34 areas. ## Columns @@ -15,7 +15,7 @@ Every scheduled job this service runs: **136 `@DfxCron` declarations** across 95 ## Scopes `scope` is a mandatory parameter of `@DfxCron` and says which process registers the job: -116 are `worker`, 7 are `api`, 13 are `both`. `CRON_ROLE` decides what a process is +116 are `worker`, 7 are `api`, 14 are `both`. `CRON_ROLE` decides what a process is (`worker`, `api`, or `all` for a single-process setup); a process runs its own scope plus `both`. `worker` is the normal case — anything writing to the database or driving business forward belongs @@ -30,16 +30,18 @@ request path loads on demand, and a job may refresh it but must not be the only ## Flags -116 of the 136 jobs carry a `process` flag, 20 do not. A job with a flag can be switched off +116 of the 137 jobs carry a `process` flag, 21 do not. A job with a flag can be switched off without a deploy — `DfxCronService` skips it when the process appears in the disabled set, which `ProcessService` refreshes from the `disabledProcesses` setting and the `DISABLED_PROCESSES` environment variable every 30 seconds. -A job **without** a flag runs unconditionally. That is deliberate for four of them — the +A job **without** a flag runs unconditionally. That is deliberate for five of them. The four `ProcessService::resync*` jobs maintain the disabled set, the JWT denylists and the staff clearance allowlist themselves, so making them switchable would let a configuration change -disable the mechanism that reads configuration changes. For the remaining 16 it is simply an -omission: +disable the mechanism that reads configuration changes. `DfxCronService::reportRole` is the role +heartbeat the `dfx-api-role-mismatch` alert reads: switched off, it would look exactly like a +process that stopped reporting, which is the condition the alert exists to catch. For the +remaining 16 it is simply an omission: | Job | Interval | | --- | --- | @@ -63,7 +65,7 @@ New jobs should declare a flag unless there is a reason like the one above. | 30 seconds | 9 | | minute | 52 | | 5 minutes | 17 | -| 10 minutes | 15 | +| 10 minutes | 16 | | hour | 16 | | day at 3am | 1 | | day at 4am | 3 | @@ -85,9 +87,9 @@ Jobs by area: | `subdomains/supporting/payin` | 12 | — | | `integration/blockchain` | 6 | — | | `subdomains/core/buy-crypto` | 6 | 4 | +| `shared/services` | 5 | 5 | | `subdomains/core/sell-crypto` | 5 | 2 | | `subdomains/supporting/payment` | 5 | 1 | -| `shared/services` | 4 | 4 | | `subdomains/core/payment-link` | 4 | — | | `subdomains/generic/kyc` | 4 | — | | `subdomains/supporting/bank` | 4 | — | @@ -119,11 +121,14 @@ Jobs by area: Every `@DfxCron(` occurrence in `src/**/*.ts`. Decorator arguments are read by a balanced-paren scan, so multi-line declarations are included — a line-based match misses 26 of them. Interval, flag and scope come from those arguments, so all three are as accurate as the source. The parsed -count is asserted against a raw text count of the decorator: **136 = 136**, no gap. Class and +count is asserted against a raw text count of the decorator: **137 = 137**, no gap. Class and method come from the enclosing `export class` (including `export abstract class`) and the identifier following the decorator. -## Known discrepancy +## Known discrepancies + +Both come from the same place: this list counts **declarations**, while `DfxCronService` registers +what `DiscoveryService.getProviders()` hands it. The two are not the same set. `CitreaBaseStrategy::checkPayInEntries` is declared on an **abstract** class. NestJS discovers providers rather than classes, so such a job is registered once per concrete subclass, not once per @@ -131,6 +136,17 @@ declaration. There is currently one subclass, so the runtime count equals the de but a second subclass would silently add another registration of the same job, sharing the flag and the interval while running as an independent timer with its own lock. +`ExchangeController::checkTrades` is **never registered**. Its class is listed under `controllers:` +in `ExchangeModule` and nowhere under `providers:`, and `getProviders()` does not return +controllers — so the scan never sees the decorator. This predates the process split and is +unchanged by it; the job has never run. `TransactionController::checkLists` looks like the same +case but is not: `HistoryModule` lists that class under both `controllers:` and `providers:`, so +the job is registered — on the provider instance, which is a different object from the controller +instance the request handlers use. + +Resolving either one is a decision about the jobs, not about this inventory, so both are recorded +here rather than fixed in passing. Of the 137 declarations, 136 have a registration path. + ## Jobs | Interval | Flag | Scope | Job | File | @@ -223,6 +239,7 @@ the interval while running as an independent timer with its own lock. | 5 minutes | `WEBHOOK` | `worker` | `WebhookNotificationService::sendWebhooks` | `subdomains/generic/user/services/webhook/webhook-notification.service.ts` | | 10 minutes | `BANK_ACCOUNT` | `worker` | `BankAccountService::reloadUncheckedBankAccounts` | `subdomains/supporting/bank/bank-account/bank-account.service.ts` | | 10 minutes | `DEURO_LOG_INFO` | `worker` | `DEuroService::processLogInfo` | `integration/blockchain/deuro/deuro.service.ts` | +| 10 minutes | — | `both` | `DfxCronService::reportRole` | `shared/services/dfx-cron.service.ts` | | 10 minutes | `MONITORING` | `worker` | `ExchangeObserver::fetch` | `subdomains/core/monitoring/observers/exchange.observer.ts` | | 10 minutes | `MONITORING` | `worker` | `ExternalServicesObserver::fetch` | `subdomains/core/monitoring/observers/external-services.observer.ts` | | 10 minutes | `BLOCKCHAIN_FEE_UPDATE` | `worker` | `FeeService::updateBlockchainFees` | `subdomains/supporting/payment/services/fee.service.ts` | diff --git a/src/shared/services/__tests__/dfx-cron.service.spec.ts b/src/shared/services/__tests__/dfx-cron.service.spec.ts index e851f3bf99..06240fba62 100644 --- a/src/shared/services/__tests__/dfx-cron.service.spec.ts +++ b/src/shared/services/__tests__/dfx-cron.service.spec.ts @@ -128,4 +128,41 @@ describe('DfxCronService', () => { jest.clearAllMocks(); } }); + + describe('role heartbeat', () => { + // The alert dfx-api-role-mismatch decides from this line which role each process is running. + // Everything it needs has to be IN the line and the line has to appear in both processes — + // the three tests below pin exactly that, because none of it is visible at the call site. + + it('runs in every process, so neither one is invisible to the alert', () => { + // Were this scoped `worker`, the API process would stop reporting and the alert could no + // longer distinguish "runs the wrong role" from "reports nothing". + const params: DfxCronParams = Reflect.getMetadata(DFX_CRONJOB_PARAMS, DfxCronService.prototype.reportRole); + + expect(params.scope).toEqual(CronScope.BOTH); + }); + + it('cannot be switched off, so a missing line always means a sick process', () => { + // With a `process` flag, a disabled watchdog would look exactly like a process that stopped + // writing the line. + const params: DfxCronParams = Reflect.getMetadata(DFX_CRONJOB_PARAMS, DfxCronService.prototype.reportRole); + + expect(params.process).toBeUndefined(); + }); + + it('names the role this process is actually running', () => { + process.env.CRON_ROLE = 'worker'; + new ConfigService(GetConfig()); + + const { service } = buildService(configuredJobs); + service.onModuleInit(); + + const info = jest.spyOn(service['logger'], 'info'); + service.reportRole(); + + // The role is what the alert matches on; the count tells the reader on call whether the + // process registered a plausible number of jobs or nearly none. + expect(info).toHaveBeenCalledWith('CronRole worker: heartbeat, 3 jobs registered'); + }); + }); }); diff --git a/src/shared/services/dfx-cron.service.ts b/src/shared/services/dfx-cron.service.ts index a0d399402a..7f1bd31549 100644 --- a/src/shared/services/dfx-cron.service.ts +++ b/src/shared/services/dfx-cron.service.ts @@ -4,7 +4,7 @@ import { CronExpression, SchedulerRegistry } from '@nestjs/schedule'; import { CronJob } from 'cron'; import { Config, CronRole } from 'src/config/config'; import { DisabledProcess } from 'src/shared/services/process.service'; -import { CronScope, DFX_CRONJOB_PARAMS, DfxCronExpression, DfxCronParams } from 'src/shared/utils/cron'; +import { CronScope, DFX_CRONJOB_PARAMS, DfxCron, DfxCronExpression, DfxCronParams } from 'src/shared/utils/cron'; import { LockClass } from 'src/shared/utils/lock'; import { Util } from 'src/shared/utils/util'; import { CustomCronExpression } from '../utils/custom-cron-expression'; @@ -21,6 +21,8 @@ interface CronJobData { export class DfxCronService implements OnModuleInit { private readonly logger = new DfxLogger(DfxCronService); + private registeredCount = 0; + constructor( private readonly discovery: DiscoveryService, private readonly metadataScanner: MetadataScanner, @@ -69,9 +71,29 @@ export class DfxCronService implements OnModuleInit { .map((scope) => `${scope}: ${registered.filter((s) => s === scope).length}`) .join(', '); + this.registeredCount = registered.length; + this.logger.info(`CronRole ${Config.cronRole}: registered ${registered.length} of ${total} jobs (${byScope})`); } + /** + * The line above says which role this process STARTED with. It cannot answer whether the two + * processes are running the right roles right now: it is written once, so an alert built on a + * counting window over it either reports nothing after the window passes, or reports permanently. + * This line answers the same question continuously, and the alert reads it. + * + * Deliberately `both`: it has to appear in EVERY process, and it carries the role, so a swapped + * assignment shows up as a wrong role rather than only as a missing line. + * + * Deliberately without a `process` flag: a watchdog that can be switched off looks, once it is + * off, exactly like a process that stopped writing the line — the alert could not tell the two + * apart. The job holds no state and does nothing but log, so there is nothing to switch off. + */ + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.BOTH }) + reportRole(): void { + this.logger.info(`CronRole ${Config.cronRole}: heartbeat, ${this.registeredCount} jobs registered`); + } + /** * `all` runs everything, which is the single-process mode. The other two roles each run their * own scope plus `both`, so a job maintaining process-local state that requests read is From db55485e81f489bd75cdf6eaaaa6bcb0615726df Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:53 +0200 Subject: [PATCH 32/86] test(cron): den Heartbeat-Test an die echte Registrierung binden MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ein Review hat gezeigt, was die drei Tests NICHT abdeckten: sie lasen Decorator-Metadaten und blieben gruen, wenn der Scan die Methode nie sieht — der Alarm haette dann geschwiegen, ohne dass ein Test faellt. Der Test uebergibt dem Scan jetzt die eigene Service-Instanz, so wie Nest es tut, und erwartet den Heartbeat in der Zaehlung. Damit das greift, liest der MetadataScanner-Mock die Prototyp-Kette: die Test-Dubletten tragen ihren Job als eigenen Schluessel, eine echte Service-Klasse auf dem Prototyp — vorher war ein Job auf einer echten Klasse fuer diese Suite unsichtbar. Ein vierter Test pinnt den Wortlaut gegen den Regexp der Alarmregel. Das ist die einzige Stelle, die beide koppelt: driftet die Meldung, faellt in diesem Repo nichts um, der Alarm wird nur still. Ausserdem im Guard-Test festgehalten, dass die setTimeout-Luecke nicht theoretisch ist: ScryptService.scheduleCatchUpRetry und ScryptWebSocketConnection.scheduleReconnect planen sich selbst neu und sind hier unsichtbar. Sie bleiben bewusst ungebunden — ein Request-Pfad erreicht sie, beide Prozesse brauchen ihren eigenen Socket und Cache. --- .../__tests__/cron-registration.guard.spec.ts | 8 ++++ .../__tests__/dfx-cron.service.spec.ts | 43 ++++++++++++++++--- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/src/shared/services/__tests__/cron-registration.guard.spec.ts b/src/shared/services/__tests__/cron-registration.guard.spec.ts index 5f8b4f430b..913e87c9a4 100644 --- a/src/shared/services/__tests__/cron-registration.guard.spec.ts +++ b/src/shared/services/__tests__/cron-registration.guard.spec.ts @@ -16,6 +16,14 @@ const SRC = join(__dirname, '..', '..', '..'); * aliased import or a scheduler reached through an object property passes unseen. Catching those * needs an AST-based rule rather than a text match. What this covers is the shape the four cases * in this repository actually had. + * + * The setTimeout gap is not hypothetical. ScryptService.scheduleCatchUpRetry and + * ScryptWebSocketConnection.scheduleReconnect both re-arm themselves and are invisible here. They + * are deliberately left alone: their state is the process-local cache and socket of the process + * they run in, and a request path reaches them (ExchangeController injects ExchangeRegistryService + * and ExchangeTxService), so both processes need their own. Binding them to a role would break the + * exchange endpoints on the API process. Anyone extending this check should read that case first — + * "the check does not see it" and "it must not be scoped" are two different statements. */ const FORBIDDEN: { pattern: RegExp; what: string; instead: string }[] = [ { diff --git a/src/shared/services/__tests__/dfx-cron.service.spec.ts b/src/shared/services/__tests__/dfx-cron.service.spec.ts index 06240fba62..0b2d47c7fb 100644 --- a/src/shared/services/__tests__/dfx-cron.service.spec.ts +++ b/src/shared/services/__tests__/dfx-cron.service.spec.ts @@ -36,7 +36,18 @@ function buildService(providers: { instance: object }[]): { >, }); const metadataScanner = createMock({ - getAllMethodNames: (instance: object) => Object.keys(instance), + // Nest walks the prototype chain. The plain test doubles carry their job as an own key, a real + // service class carries it on its prototype — both have to be visible here, otherwise a job + // declared on an actual service would be invisible to this suite while every assertion passes. + getAllMethodNames: (instance: object) => { + const proto = Object.getPrototypeOf(instance); + const inherited = + proto && proto !== Object.prototype + ? Object.getOwnPropertyNames(proto).filter((name) => name !== 'constructor') + : []; + + return [...Object.keys(instance), ...inherited]; + }, }); const scheduler = createMock(); @@ -150,19 +161,41 @@ describe('DfxCronService', () => { expect(params.process).toBeUndefined(); }); - it('names the role this process is actually running', () => { + it('names the role this process is actually running, and counts itself', () => { process.env.CRON_ROLE = 'worker'; new ConfigService(GetConfig()); + // The service is handed its OWN instance among the providers, the way Nest does it: the job + // lives on DfxCronService itself, so a scan that skipped it would leave the heartbeat + // unregistered while every metadata assertion above still passed. + const { service } = buildService(configuredJobs); + const { service: scanned } = buildService([...configuredJobs, { instance: service }]); + scanned.onModuleInit(); + + const info = jest.spyOn(scanned['logger'], 'info'); + scanned.reportRole(); + + // Three worker/both jobs plus reportRole itself. The role is what the alert matches on; the + // count tells the reader on call whether the process registered a plausible number of jobs. + expect(info).toHaveBeenCalledWith('CronRole worker: heartbeat, 4 jobs registered'); + }); + + it('produces a line the alert query actually matches', () => { + // The alert reads this line with `CronRole (api|worker|all): heartbeat, [0-9]+ jobs + // registered`. Pinning the wording here is the only place that couples the two: nothing in + // this repository fails if the message drifts, the alert just goes quiet. + process.env.CRON_ROLE = 'api'; + new ConfigService(GetConfig()); + const { service } = buildService(configuredJobs); service.onModuleInit(); const info = jest.spyOn(service['logger'], 'info'); service.reportRole(); - // The role is what the alert matches on; the count tells the reader on call whether the - // process registered a plausible number of jobs or nearly none. - expect(info).toHaveBeenCalledWith('CronRole worker: heartbeat, 3 jobs registered'); + const line = info.mock.calls[0][0] as string; + + expect(line).toMatch(/CronRole (api|worker|all): heartbeat, [0-9]+ jobs registered/); }); }); }); From a731a1baa69e88d029460c98ed9454c05f4e21cf Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:54 +0200 Subject: [PATCH 33/86] fix(test): den Scanner-Mock auf Methoden einschraenken MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Der erweiterte Mock lieferte auch eigene Felder der Instanz. Bei einer echten Service-Klasse sind das die injizierten Abhaengigkeiten und Zustandsfelder — Reflect.getMetadata auf einer Zahl wirft einen TypeError, statt undefined zu liefern, und riss die ganze Suite mit. Der Filter bildet jetzt ab, was der echte MetadataScanner tut: nur Methoden. Gefunden von der CI, nicht von mir — lokal fehlt node_modules, Tests sind hier nicht ausfuehrbar. --- src/shared/services/__tests__/dfx-cron.service.spec.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/shared/services/__tests__/dfx-cron.service.spec.ts b/src/shared/services/__tests__/dfx-cron.service.spec.ts index 0b2d47c7fb..c45f7031d3 100644 --- a/src/shared/services/__tests__/dfx-cron.service.spec.ts +++ b/src/shared/services/__tests__/dfx-cron.service.spec.ts @@ -46,7 +46,12 @@ function buildService(providers: { instance: object }[]): { ? Object.getOwnPropertyNames(proto).filter((name) => name !== 'constructor') : []; - return [...Object.keys(instance), ...inherited]; + // Methods only, like the real scanner. Own keys of a service instance are its injected + // dependencies and fields — handing those to the caller makes it read metadata off a number, + // which throws rather than returning undefined. + return [...Object.keys(instance), ...inherited].filter( + (name) => typeof (instance as Record)[name] === 'function', + ); }, }); const scheduler = createMock(); From ddedf49c9dd6b67f2362d2c6d2d7b1748096162b Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:56 +0200 Subject: [PATCH 34/86] feat(cron): einen Lease in der Datenbank gegen doppelt laufende Jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bisher hielt nur LockClass die Jobs auseinander, und ihr Zustand ist ein Feld im Prozessspeicher — sie kann einen zweiten Prozess nicht sehen. Das genuegte, solange die API ein Prozess war. Mit der Trennung von HTTP-Prozess und Worker ruht "genau ein Prozess faehrt diesen Job" auf der Konfiguration, einem Satz im Runbook und einem Alarm, der einen Doppellauf eine Viertelstunde spaeter MELDET. Fuer einen Pfad, der Geld bewegt, ist Erkennung die zweitbeste Antwort. Jobs mit Scope worker oder api halten jetzt fuer die Dauer ihres Laufs einen Lease in der Tabelle cron_lease. Ein einziges Statement entscheidet darueber: der Upsert uebernimmt die Zeile nur, wenn sie abgelaufen ist — zwei Prozesse im Rennen serialisiert der Primaerschluessel, genau einer bekommt eine Zeile zurueck. Damit ist der Doppellauf ausgeschlossen, statt gemeldet: bei einem ausgebliebenen Recreate, bei einem zweiten Worker aus --scale, bei zwei Prozessen auf 'all' nach einem Rueckbau. Scope both ist bewusst ausgenommen. Diese Jobs pflegen Zustand, den ein Request-Pfad im JEWEILIGEN Prozess liest; ein Lease darueber liesse den Verlierer des Rennens verhungern und den Zustand einfrieren. Ihre Sicherheit kommt aus einer anderen Eigenschaft: doppelt zu laufen muss bei ihnen von Bauart her harmlos sein. Ein Lease statt eines Advisory Locks, weil letzterer an die Verbindung gebunden ist: er hielte eine Poolverbindung ueber die ganze Laufzeit, und 67 Jobs deklarieren einen Timeout in Minutenhoehe. Gegen SQL_POOL_MAX=40 ist das ein reales Risiko fuers Verbindungsbudget. WAS ER NICHT KANN, und das steht so auch im Code: Faellt die Datenbank waehrend eines Laufs aus, laesst sich der Lease nicht verlaengern und laeuft ab — ein zweiter Prozess koennte den Job dann starten, waehrend der erste noch arbeitet. Vollstaendiges Fencing braeuchte ein Token an jedem einzelnen Schreibzugriff. Was der Lease leistet, ist die Verkuerzung des Fensters von "unbegrenzt, bis jemand den Alarm liest" auf die Lease-Dauer. Ein verlorener Lease wird als Fehler geloggt. Bei unerreichbarer Datenbank laeuft der Job NICHT. Ein Job, der Geld bewegt, darf nicht auf der Annahme weiterarbeiten, er sei vermutlich allein — und genau dann ist diese Annahme am wenigsten tragfaehig. Sieben Tests fuer den Lease, zwei fuer die Einbindung. Drei Mutationen gegengeprueft (Lease auch fuer both, Ablaufbedingung entfernt, bei DB-Fehler trotzdem laufen): jede faellt auf. Ausserdem faehrt der Rollen-Heartbeat jetzt mit useDelay: false. Der Alarm liest ihn ueber ein 12-Minuten-Fenster; mit dem Standard-Jitter betraegt der groesste Abstand zweier Zeilen 660 s, die Marge also 60 s — und der Jitter ist ueber CRON_JOB_DELAY von aussen verstellbar. Ein Waechter darf seine Taktung nicht an einem Regler haengen haben, der zum Lastverteilen gedacht ist. Der Guard-Test bekommt eine zweite Ausnahme: die Lease-Verlaengerung ist an die Lebensdauer EINES Laufs gebunden, nicht an einen Zeitplan, und sie durch @DfxCron zu fuehren waere zirkulaer — sie ist der Mechanismus, der @DfxCron-Jobs vor dem Doppellauf schuetzt. --- CONTRIBUTING.md | 6 + migration/1785600000000-AddCronLease.js | 43 ++++++ .../__tests__/cron-lease.service.spec.ts | 130 +++++++++++++++++ .../__tests__/cron-registration.guard.spec.ts | 8 +- .../__tests__/dfx-cron.service.spec.ts | 81 ++++++++++- src/shared/services/cron-lease.service.ts | 135 ++++++++++++++++++ src/shared/services/dfx-cron.service.ts | 43 +++++- src/shared/shared.module.ts | 2 + 8 files changed, 444 insertions(+), 4 deletions(-) create mode 100644 migration/1785600000000-AddCronLease.js create mode 100644 src/shared/services/__tests__/cron-lease.service.spec.ts create mode 100644 src/shared/services/cron-lease.service.ts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9152159cfe..d19cf14bbb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -538,6 +538,12 @@ outside the process infers health from the job still running — a watchdog that looks, once it is off, exactly like the failure it watches for. Without a flag the job runs unconditionally and cannot be switched off without a deploy. +A job scoped `worker` or `api` additionally holds a **lease in the database** for the duration of +its run (`CronLeaseService`), so it runs in at most one process even if the configuration is +wrong — a missed recreate, a second worker from `--scale`, two processes on `all` after a +rollback. The in-process lock cannot see any of those. Jobs scoped `both` are exempt by design: +they must run everywhere, which is why running them twice has to be harmless by construction. + [docs/cron-jobs.md](docs/cron-jobs.md) lists every scheduled job with its interval, flag and scope. **Adding, removing or re-scheduling a job must be reflected there in the same PR.** diff --git a/migration/1785600000000-AddCronLease.js b/migration/1785600000000-AddCronLease.js new file mode 100644 index 0000000000..65064be081 --- /dev/null +++ b/migration/1785600000000-AddCronLease.js @@ -0,0 +1,43 @@ +/** + * @typedef {import('typeorm').MigrationInterface} MigrationInterface + * @typedef {import('typeorm').QueryRunner} QueryRunner + */ + +/** + * Cross-process lease for scheduled jobs. + * + * Until now the only thing stopping a job from running twice was `LockClass`, and its state is a + * field in process memory — it cannot see a second process. That was acceptable while the API ran + * as a single process. With the HTTP process and the worker split apart, "exactly one process runs + * this job" became an assumption held up by configuration, a runbook sentence and an alert that + * *reports* a double run about fifteen minutes after it starts. For a path that moves money, + * detection is the second-best answer. + * + * This table makes it structural: a job scoped to exactly one process must hold a row here for the + * duration of its run, and the row is claimable by only one process at a time. + * + * No foreign keys, deliberately. The table is infrastructure, not domain data, and PRD carries + * tables without a primary key from the MSSQL cutover — a FK into one of them would fail at boot. + * `name` is the primary key, so the claim is a single atomic upsert with no index to keep in sync. + * + * @class @implements {MigrationInterface} + */ +module.exports = class AddCronLease1785600000000 { + name = 'AddCronLease1785600000000'; + + /** + * @param {QueryRunner} queryRunner + */ + async up(queryRunner) { + await queryRunner.query( + `CREATE TABLE "cron_lease" ("name" character varying(256) NOT NULL, "owner" character varying(256) NOT NULL, "acquired" TIMESTAMP NOT NULL DEFAULT now(), "expires" TIMESTAMP NOT NULL, CONSTRAINT "PK_cron_lease_name" PRIMARY KEY ("name"))`, + ); + } + + /** + * @param {QueryRunner} queryRunner + */ + async down(queryRunner) { + await queryRunner.query(`DROP TABLE "cron_lease"`); + } +}; diff --git a/src/shared/services/__tests__/cron-lease.service.spec.ts b/src/shared/services/__tests__/cron-lease.service.spec.ts new file mode 100644 index 0000000000..33e2f789bb --- /dev/null +++ b/src/shared/services/__tests__/cron-lease.service.spec.ts @@ -0,0 +1,130 @@ +import { createMock } from '@golevelup/ts-jest'; +import { ConfigService, GetConfig } from 'src/config/config'; +import { DataSource } from 'typeorm'; +import { CronLeaseService } from '../cron-lease.service'; + +/** + * The lease is the only thing that stops a payout from running twice when two processes disagree + * about who owns a job. Every test here is written from that angle: not "does the method return + * true", but "can this state let the task run twice, or stop it from running at all". + */ +describe('CronLeaseService', () => { + const original = process.env.CRON_ROLE; + + /** Mirrors the two shapes `DataSource.query` returns: rows for INSERT..RETURNING, [rows, count] for UPDATE. */ + function buildService(responses: { acquire?: unknown[]; renew?: [unknown[], number]; onQuery?: jest.Mock }) { + const onQuery = + responses.onQuery ?? + jest.fn().mockImplementation((sql: string) => { + if (sql.includes('INSERT INTO')) return Promise.resolve(responses.acquire ?? [{ owner: 'x' }]); + if (sql.includes('UPDATE')) return Promise.resolve(responses.renew ?? [[], 1]); + return Promise.resolve([]); + }); + + return { service: new CronLeaseService(createMock({ query: onQuery })), onQuery }; + } + + beforeEach(() => { + process.env.CRON_ROLE = 'worker'; + new ConfigService(GetConfig()); + }); + + afterEach(() => { + jest.clearAllMocks(); + + if (original == null) delete process.env.CRON_ROLE; + else process.env.CRON_ROLE = original; + + new ConfigService(GetConfig()); + }); + + it('runs the task when it holds the lease', async () => { + const { service } = buildService({ acquire: [{ owner: 'worker:1' }] }); + const task = jest.fn().mockResolvedValue(undefined); + + await service.run('SomeService::job', 60, task); + + expect(task).toHaveBeenCalledTimes(1); + }); + + it('does NOT run the task when another process holds the lease', async () => { + // The claim statement returns no row when an unexpired lease belongs to someone else. This is + // the case the whole mechanism exists for: the second process must stay out. + const { service } = buildService({ acquire: [] }); + const task = jest.fn().mockResolvedValue(undefined); + + await service.run('SomeService::job', 60, task); + + expect(task).not.toHaveBeenCalled(); + }); + + it('does NOT run the task when the database is unreachable', async () => { + // Fail-closed on purpose. A job that moves money must not proceed on the assumption that it is + // probably alone — an unreachable database is exactly when that assumption is least safe. + const onQuery = jest.fn().mockRejectedValue(new Error('connection refused')); + const { service } = buildService({ onQuery }); + const task = jest.fn().mockResolvedValue(undefined); + + await service.run('SomeService::job', 60, task); + + expect(task).not.toHaveBeenCalled(); + }); + + it('releases the lease even when the task throws', async () => { + // Without this an error would leave the row behind, and the job would sit out every cycle + // until the lease expired — a silent outage of that job. + const { service, onQuery } = buildService({}); + const task = jest.fn().mockRejectedValue(new Error('job blew up')); + + await expect(service.run('SomeService::job', 60, task)).rejects.toThrow('job blew up'); + + expect(onQuery.mock.calls.some(([sql]) => (sql as string).includes('DELETE FROM'))).toBe(true); + }); + + it('claims only a lease that has expired, never one that is still held', async () => { + // The safety of the whole thing rests on this single statement, so its shape is pinned here: + // an upsert whose update branch is conditional on expiry. Drop the WHERE and two processes + // would happily take turns owning the same job. + const { service, onQuery } = buildService({}); + + await service.acquire('SomeService::job', 60); + + const sql = (onQuery.mock.calls[0][0] as string).replace(/\s+/g, ' '); + + expect(sql).toContain('ON CONFLICT ("name") DO UPDATE'); + expect(sql).toContain('WHERE "cron_lease"."expires" <= now()'); + expect(sql).toContain('RETURNING "owner"'); + }); + + it('scopes renewal and release to this process', async () => { + // A run that already lost its lease must not be able to extend or delete the row a different + // process now owns. + const { service, onQuery } = buildService({}); + + await service.renew('SomeService::job', 60); + await service.release('SomeService::job'); + + const [renewSql] = onQuery.mock.calls[0]; + const [releaseSql] = onQuery.mock.calls[1]; + + expect((renewSql as string).replace(/\s+/g, ' ')).toContain('WHERE "name" = $1 AND "owner" = $2'); + expect((releaseSql as string).replace(/\s+/g, ' ')).toContain('WHERE "name" = $1 AND "owner" = $2'); + }); + + it('gives two processes of the same role different owners', async () => { + // The role alone would let a restarted container renew the lease its predecessor took. The + // random part is what makes the owner identify a process rather than a kind of process. + const { service: first, onQuery: firstQuery } = buildService({}); + const { service: second, onQuery: secondQuery } = buildService({}); + + await first.acquire('SomeService::job', 60); + await second.acquire('SomeService::job', 60); + + const firstOwner = firstQuery.mock.calls[0][1][1] as string; + const secondOwner = secondQuery.mock.calls[0][1][1] as string; + + expect(firstOwner.startsWith('worker:')).toBe(true); + expect(secondOwner.startsWith('worker:')).toBe(true); + expect(firstOwner).not.toEqual(secondOwner); + }); +}); diff --git a/src/shared/services/__tests__/cron-registration.guard.spec.ts b/src/shared/services/__tests__/cron-registration.guard.spec.ts index 913e87c9a4..4dc61bcce5 100644 --- a/src/shared/services/__tests__/cron-registration.guard.spec.ts +++ b/src/shared/services/__tests__/cron-registration.guard.spec.ts @@ -47,7 +47,13 @@ const FORBIDDEN: { pattern: RegExp; what: string; instead: string }[] = [ * A timer tied to the lifetime of an object rather than to a schedule. It is bound to the role * directly, which is the same decision the scope expresses for a cron job. */ -const ALLOWED = ['integration/blockchain/spark/spark-client.ts']; +const ALLOWED = [ + 'integration/blockchain/spark/spark-client.ts', + // The lease renewal is bound to the lifetime of a single job run, not to a schedule: it starts + // when that run takes the lease and is cleared in its `finally`. Routing it through @DfxCron + // would be circular — it is the mechanism that keeps @DfxCron jobs from running twice. + 'shared/services/cron-lease.service.ts', +]; function sourceFiles(dir: string): string[] { return readdirSync(dir).flatMap((entry) => { diff --git a/src/shared/services/__tests__/dfx-cron.service.spec.ts b/src/shared/services/__tests__/dfx-cron.service.spec.ts index c45f7031d3..98c438879b 100644 --- a/src/shared/services/__tests__/dfx-cron.service.spec.ts +++ b/src/shared/services/__tests__/dfx-cron.service.spec.ts @@ -9,6 +9,8 @@ import { DiscoveryService, MetadataScanner } from '@nestjs/core'; import { CronExpression, SchedulerRegistry } from '@nestjs/schedule'; import { ConfigService, GetConfig } from 'src/config/config'; import { CronScope, DFX_CRONJOB_PARAMS, DfxCronParams } from 'src/shared/utils/cron'; +import { CronJob } from 'cron'; +import { CronLeaseService } from '../cron-lease.service'; import { DfxCronService } from '../dfx-cron.service'; import { Process } from '../process.service'; @@ -55,8 +57,13 @@ function buildService(providers: { instance: object }[]): { }, }); const scheduler = createMock(); + // Runs the task straight through: these tests are about which jobs get registered, not + // about the lease. What the lease itself does has its own suite. + const leases = createMock({ + run: (_job: string, _ttl: number, task: () => Promise) => task(), + }); - return { service: new DfxCronService(discovery, metadataScanner, scheduler), scheduler }; + return { service: new DfxCronService(discovery, metadataScanner, scheduler, leases), scheduler }; } describe('DfxCronService', () => { @@ -145,6 +152,78 @@ describe('DfxCronService', () => { } }); + describe('cross-process lease', () => { + // Which process runs a job is decided by configuration, and configuration can be wrong. The + // lease is what makes a wrong configuration harmless instead of expensive — these two tests + // pin who goes through it, because nothing at the call site shows it. + + /** Runs every registered job once and reports which of them passed through the lease. */ + async function leasedJobs(role: string): Promise { + process.env.CRON_ROLE = role; + new ConfigService(GetConfig()); + + const seen: string[] = []; + // Own job set: no `process` flag (a disabled one would be skipped before the lease is even + // reached) and `useDelay: false` (the real delay is up to a minute). + const jobs = [ + providerWithJob('workerJob', { + expression: CronExpression.EVERY_MINUTE, + scope: CronScope.WORKER, + useDelay: false, + }), + providerWithJob('workerJobWithoutProcess', { + expression: CronExpression.EVERY_MINUTE, + scope: CronScope.WORKER, + useDelay: false, + }), + providerWithJob('bothJob', { + expression: CronExpression.EVERY_MINUTE, + scope: CronScope.BOTH, + useDelay: false, + }), + ]; + const discovery = createMock({ + getProviders: () => + jobs.map((p) => ({ ...p, isDependencyTreeStatic: () => true })) as ReturnType< + DiscoveryService['getProviders'] + >, + }); + const metadataScanner = createMock({ getAllMethodNames: (i: object) => Object.keys(i) }); + const leaseSpy = createMock({ + run: (job: string, _ttl: number, task: () => Promise) => { + seen.push(job); + return task(); + }, + }); + const registry = createMock(); + + new DfxCronService(discovery, metadataScanner, registry, leaseSpy).onModuleInit(); + + // The CronJob constructor is mocked, so the scheduled function is the second argument. + const scheduled = (CronJob as unknown as jest.Mock).mock.calls.map(([, fn]) => fn as () => unknown); + for (const fire of scheduled) await fire(); + + return seen; + } + + it('sends single-process jobs through the lease', async () => { + // Without this a second worker — from `--scale`, a missed recreate, a rollback — would run + // every one of these a second time, and the in-process lock cannot see it. + const leased = await leasedJobs('worker'); + + expect(leased).toContain('Object::workerJob'); + expect(leased).toContain('Object::workerJobWithoutProcess'); + }); + + it('lets jobs scoped both run WITHOUT a lease', async () => { + // These maintain state a request path on THIS process reads, so they must run everywhere. A + // lease over them would starve whichever process lost the race and freeze that state. + const leased = await leasedJobs('worker'); + + expect(leased).not.toContain('Object::bothJob'); + }); + }); + describe('role heartbeat', () => { // The alert dfx-api-role-mismatch decides from this line which role each process is running. // Everything it needs has to be IN the line and the line has to appear in both processes — diff --git a/src/shared/services/cron-lease.service.ts b/src/shared/services/cron-lease.service.ts new file mode 100644 index 0000000000..53a5f281ce --- /dev/null +++ b/src/shared/services/cron-lease.service.ts @@ -0,0 +1,135 @@ +import { Injectable } from '@nestjs/common'; +import { randomUUID } from 'crypto'; +import { Config } from 'src/config/config'; +import { DataSource } from 'typeorm'; +import { DfxLogger } from './dfx-logger'; + +/** + * A lease on a scheduled job, held in the database for as long as the job runs. + * + * `LockClass` keeps its state in a field of a process-local object. That was enough while the API + * ran as one process; it cannot see a second one. Since the HTTP process and the worker are split + * apart, "exactly one process runs this job" rests on configuration, a runbook sentence and an + * alert that *reports* a double run after the fact. For a path that moves money that is the + * second-best answer, so this makes it structural. + * + * The lease is claimed per job name, and only one owner can hold it. It carries an expiry rather + * than a lock held on a connection: a connection-bound `pg_advisory_lock` would occupy one pooled + * connection for the whole runtime of the job, and 67 of the jobs declare a timeout measured in + * minutes. Against `SQL_POOL_MAX=40` that is a real risk to the connection budget. An expiring row + * costs one short query to take, one to extend, one to release. + * + * **What it does not do.** If the database becomes unreachable while a job runs, the lease cannot + * be extended and eventually expires — a second process may then start the same job while the + * first is still working. Preventing that outright would require every write inside every job to + * carry the lease token, which this does not attempt. What it does is turn an unbounded window + * ("until a human reads the alert") into a bounded one (the lease duration). A lost lease is + * logged at error level, because it means the run that is still going has no claim to the job any + * more. + */ +@Injectable() +export class CronLeaseService { + private readonly logger = new DfxLogger(CronLeaseService); + + private ownerId?: string; + + /** + * Identifies THIS process for the lifetime of the process. The role makes a stray row readable + * for an operator; the random part is what actually distinguishes two processes of the same + * role — a restarted container must not be able to renew a lease that its predecessor took. + * + * Resolved on first use rather than in a field initializer: `Config` does not exist until + * `ConfigService` has been constructed, and a provider built before it would take down the boot. + */ + private get owner(): string { + return (this.ownerId ??= `${Config.cronRole}:${randomUUID()}`); + } + + constructor(private readonly dataSource: DataSource) {} + + /** + * Claims the lease for `job`, or reports that someone else holds it. + * + * A single statement decides it: the insert either creates the row or takes it over from an + * expired owner. Two processes racing here are serialised by the primary key, so exactly one of + * them sees a returned row. Read the `WHERE` as "only if nobody is currently holding it" — an + * unexpired row belonging to another process leaves the update out and returns nothing. + */ + async acquire(job: string, ttlSeconds: number): Promise { + const claimed = await this.dataSource.query( + `INSERT INTO "cron_lease" ("name", "owner", "acquired", "expires") + VALUES ($1, $2, now(), now() + ($3 || ' seconds')::interval) + ON CONFLICT ("name") DO UPDATE + SET "owner" = EXCLUDED."owner", "acquired" = EXCLUDED."acquired", "expires" = EXCLUDED."expires" + WHERE "cron_lease"."expires" <= now() + RETURNING "owner"`, + [job, this.owner, `${ttlSeconds}`], + ); + + return claimed.length > 0; + } + + /** + * Pushes the expiry out while the job is still running. Returns false when this process is no + * longer the owner — which means another process has taken the job over and this run should be + * treated as having lost its claim. + */ + async renew(job: string, ttlSeconds: number): Promise { + const [, affected] = await this.dataSource.query( + `UPDATE "cron_lease" + SET "expires" = now() + ($3 || ' seconds')::interval + WHERE "name" = $1 AND "owner" = $2`, + [job, this.owner, `${ttlSeconds}`], + ); + + return affected > 0; + } + + /** + * Releases the lease. Scoped to this owner so a run that already lost the lease cannot delete + * the row a different process is now holding. + */ + async release(job: string): Promise { + await this.dataSource.query(`DELETE FROM "cron_lease" WHERE "name" = $1 AND "owner" = $2`, [job, this.owner]); + } + + /** + * Runs `task` only if this process can claim the lease, and keeps the claim alive meanwhile. + * + * Failing to reach the database means NOT running: a job that moves money must not proceed on + * the assumption that it is probably alone. The caller sees the same outcome as a job whose + * lease is held elsewhere — it simply does not run this cycle and tries again on the next. + */ + async run(job: string, ttlSeconds: number, task: () => Promise): Promise { + let acquired: boolean; + try { + acquired = await this.acquire(job, ttlSeconds); + } catch (e) { + this.logger.error(`Skipping ${job}: could not reach the lease table`, e); + return; + } + + if (!acquired) return; + + // Renew at a third of the lease so two consecutive failures still leave a full attempt before + // the lease lapses. Unref'd: a pending timer must never hold the process open on shutdown. + const renewal = setInterval( + () => { + void this.renew(job, ttlSeconds) + .then((stillOurs) => { + if (!stillOurs) this.logger.error(`Lost the lease for ${job} while it was still running`); + }) + .catch((e) => this.logger.error(`Could not extend the lease for ${job}`, e)); + }, + (ttlSeconds / 3) * 1000, + ); + renewal.unref(); + + try { + await task(); + } finally { + clearInterval(renewal); + await this.release(job).catch((e) => this.logger.error(`Could not release the lease for ${job}`, e)); + } + } +} diff --git a/src/shared/services/dfx-cron.service.ts b/src/shared/services/dfx-cron.service.ts index 7f1bd31549..31e4a9eaa2 100644 --- a/src/shared/services/dfx-cron.service.ts +++ b/src/shared/services/dfx-cron.service.ts @@ -6,10 +6,14 @@ import { Config, CronRole } from 'src/config/config'; import { DisabledProcess } from 'src/shared/services/process.service'; import { CronScope, DFX_CRONJOB_PARAMS, DfxCron, DfxCronExpression, DfxCronParams } from 'src/shared/utils/cron'; import { LockClass } from 'src/shared/utils/lock'; +import { CronLeaseService } from './cron-lease.service'; import { Util } from 'src/shared/utils/util'; import { CustomCronExpression } from '../utils/custom-cron-expression'; import { DfxLogger } from './dfx-logger'; +/** Lease length for a job that declares no timeout of its own. */ +const DEFAULT_LEASE_SECONDS = 300; + interface CronJobData { instance: object; methodRef: any; @@ -27,6 +31,7 @@ export class DfxCronService implements OnModuleInit { private readonly discovery: DiscoveryService, private readonly metadataScanner: MetadataScanner, private readonly schedulerRegisty: SchedulerRegistry, + private readonly leases: CronLeaseService, ) {} onModuleInit() { @@ -89,7 +94,12 @@ export class DfxCronService implements OnModuleInit { * off, exactly like a process that stopped writing the line — the alert could not tell the two * apart. The job holds no state and does nothing but log, so there is nothing to switch off. */ - @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.BOTH }) + // `useDelay: false`: the alert reads this line over a 12-minute window. With the default jitter + // the gap between two heartbeats can reach 660 s, leaving 60 s of margin — and the jitter is + // configurable through CRON_JOB_DELAY, so someone could close that margin from the outside + // without ever seeing this code. A watchdog must not have its own timing tuned by a knob meant + // for spreading load. + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.BOTH, useDelay: false }) reportRole(): void { this.logger.info(`CronRole ${Config.cronRole}: heartbeat, ${this.registeredCount} jobs registered`); } @@ -116,8 +126,9 @@ export class DfxCronService implements OnModuleInit { const lock = LockClass.create(data.params.timeout ?? Infinity); const context = { target: data.instance.constructor.name, method: data.methodName }; - const cronJob = new CronJob(data.params.expression, () => lock(this.wrapFunction(data), context)); const cronJobName = `${context.target}::${context.method}`; + const run = this.guardAcrossProcesses(cronJobName, data); + const cronJob = new CronJob(data.params.expression, () => lock(run, context)); this.schedulerRegisty.addCronJob(cronJobName, cronJob); cronJob.start(); @@ -125,6 +136,34 @@ export class DfxCronService implements OnModuleInit { this.logger.verbose(`Registered ${cronJobName} (${data.params.scope})`); } + /** + * Wraps a job so that at most one process in the deployment runs it at a time. + * + * `lock` above only spans this process. Which process a job belongs to is decided by + * configuration, and configuration can be wrong — a missed recreate leaves the old role in + * place, `--scale` creates a second worker, a rollback puts two processes on `all`. In every + * one of those the two processes hold separate locks and every payout runs twice. The lease + * closes that by construction instead of reporting it a quarter of an hour later. + * + * `BOTH` jobs are deliberately exempt. They exist because a request path on THIS process reads + * the state they maintain, so they have to run in every process — a lease over them would + * starve whichever process lost the race, and the job would silently stop maintaining that + * state. Their safety comes from a different property: running twice must be harmless by + * construction, which is what CONTRIBUTING requires of them. + */ + private guardAcrossProcesses(cronJobName: string, data: CronJobData): () => Promise { + const task = this.wrapFunction(data); + + if (data.params.scope === CronScope.BOTH) return task; + + // The lease has to outlive a single run, so it follows the job’s own timeout where there is + // one. Where there is none the job carries no expectation of its duration either, and five + // minutes is long enough that the renewal (every third of it) keeps a healthy run alive. + const ttlSeconds = Number.isFinite(data.params.timeout) ? data.params.timeout : DEFAULT_LEASE_SECONDS; + + return () => this.leases.run(cronJobName, ttlSeconds, task); + } + private wrapFunction(data: CronJobData) { const context = { target: data.instance.constructor.name, method: data.methodName }; diff --git a/src/shared/shared.module.ts b/src/shared/shared.module.ts index 371f467022..697f7f5f32 100644 --- a/src/shared/shared.module.ts +++ b/src/shared/shared.module.ts @@ -35,6 +35,7 @@ import { Setting } from './models/setting/setting.entity'; import { SettingRepository } from './models/setting/setting.repository'; import { SettingService } from './models/setting/setting.service'; import { RepositoryFactory } from './repositories/repository.factory'; +import { CronLeaseService } from './services/cron-lease.service'; import { DfxCronService } from './services/dfx-cron.service'; import { HttpService } from './services/http.service'; import { PaymentInfoService } from './services/payment-info.service'; @@ -72,6 +73,7 @@ import { ProcessService } from './services/process.service'; PaymentInfoService, IpLogService, ProcessService, + CronLeaseService, DfxCronService, ], exports: [ From 48b1a0fcdbaa1f091c9bbc997ee5d3f1ea52f50c Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:57 +0200 Subject: [PATCH 35/86] Bound how long a dead process can block a job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The lease expiry was derived from the job's own timeout. `timeout` is measured in seconds and sixteen jobs declare 7200, so a process that died without reaching its `finally` — which every deployment causes, since nothing in this repository ever asked Nest for a shutdown hook — left the row behind for up to two hours. Its successor skipped the job for that whole window and said nothing. The expiry now answers its own question instead of the job's: it bounds how long a claim outlives an owner that can no longer speak for itself, which is a property of the failure mode and not of the work. One fixed minute, renewed every twenty seconds. Shutdown is the other half. `beforeApplicationShutdown` — before, because TypeOrmCoreModule closes the connection in `onApplicationShutdown` — waits up to ten seconds for the runs this process holds, so their normal release hands the job to the successor. A run still going after that keeps its lease: taking it away would let the successor start the same job while this process works on it for the rest of its stop grace period, which is the one outcome the lease exists to prevent. It lapses within the minute instead. Tests: a job declaring 7200 now claims for 60; the lease survives a shutdown that outlasts the grace; the shutdown does not return before the job does; and bootstrap is pinned to enable the hooks at all. --- src/main.ts | 6 + .../__tests__/cron-lease.service.spec.ts | 133 ++++++++++++++++- .../__tests__/dfx-cron.service.spec.ts | 47 +++++- src/shared/services/cron-lease.service.ts | 138 ++++++++++++++---- src/shared/services/dfx-cron.service.ts | 10 +- 5 files changed, 288 insertions(+), 46 deletions(-) diff --git a/src/main.ts b/src/main.ts index 744ea6017a..da229f83e4 100644 --- a/src/main.ts +++ b/src/main.ts @@ -81,6 +81,12 @@ async function bootstrap() { app.use(apiTraceMiddleware()); } + // Without this, Nest never runs a shutdown hook and a deployment kills the process mid-job. The + // cron lease depends on it: see CronLeaseService.beforeApplicationShutdown. Restricted to the + // two signals a deployment actually sends, rather than the full default set — the others are + // crash signals whose default handling should stay untouched. + app.enableShutdownHooks(['SIGTERM', 'SIGINT']); + app.useWebSocketAdapter(new WsAdapter(app)); app.enableVersioning({ diff --git a/src/shared/services/__tests__/cron-lease.service.spec.ts b/src/shared/services/__tests__/cron-lease.service.spec.ts index 33e2f789bb..46a3b695bd 100644 --- a/src/shared/services/__tests__/cron-lease.service.spec.ts +++ b/src/shared/services/__tests__/cron-lease.service.spec.ts @@ -1,4 +1,6 @@ import { createMock } from '@golevelup/ts-jest'; +import { readFileSync } from 'fs'; +import { join } from 'path'; import { ConfigService, GetConfig } from 'src/config/config'; import { DataSource } from 'typeorm'; import { CronLeaseService } from '../cron-lease.service'; @@ -42,7 +44,7 @@ describe('CronLeaseService', () => { const { service } = buildService({ acquire: [{ owner: 'worker:1' }] }); const task = jest.fn().mockResolvedValue(undefined); - await service.run('SomeService::job', 60, task); + await service.run('SomeService::job', task); expect(task).toHaveBeenCalledTimes(1); }); @@ -53,7 +55,7 @@ describe('CronLeaseService', () => { const { service } = buildService({ acquire: [] }); const task = jest.fn().mockResolvedValue(undefined); - await service.run('SomeService::job', 60, task); + await service.run('SomeService::job', task); expect(task).not.toHaveBeenCalled(); }); @@ -65,7 +67,7 @@ describe('CronLeaseService', () => { const { service } = buildService({ onQuery }); const task = jest.fn().mockResolvedValue(undefined); - await service.run('SomeService::job', 60, task); + await service.run('SomeService::job', task); expect(task).not.toHaveBeenCalled(); }); @@ -76,7 +78,7 @@ describe('CronLeaseService', () => { const { service, onQuery } = buildService({}); const task = jest.fn().mockRejectedValue(new Error('job blew up')); - await expect(service.run('SomeService::job', 60, task)).rejects.toThrow('job blew up'); + await expect(service.run('SomeService::job', task)).rejects.toThrow('job blew up'); expect(onQuery.mock.calls.some(([sql]) => (sql as string).includes('DELETE FROM'))).toBe(true); }); @@ -87,7 +89,7 @@ describe('CronLeaseService', () => { // would happily take turns owning the same job. const { service, onQuery } = buildService({}); - await service.acquire('SomeService::job', 60); + await service.acquire('SomeService::job'); const sql = (onQuery.mock.calls[0][0] as string).replace(/\s+/g, ' '); @@ -101,7 +103,7 @@ describe('CronLeaseService', () => { // process now owns. const { service, onQuery } = buildService({}); - await service.renew('SomeService::job', 60); + await service.renew('SomeService::job'); await service.release('SomeService::job'); const [renewSql] = onQuery.mock.calls[0]; @@ -117,8 +119,8 @@ describe('CronLeaseService', () => { const { service: first, onQuery: firstQuery } = buildService({}); const { service: second, onQuery: secondQuery } = buildService({}); - await first.acquire('SomeService::job', 60); - await second.acquire('SomeService::job', 60); + await first.acquire('SomeService::job'); + await second.acquire('SomeService::job'); const firstOwner = firstQuery.mock.calls[0][1][1] as string; const secondOwner = secondQuery.mock.calls[0][1][1] as string; @@ -127,4 +129,119 @@ describe('CronLeaseService', () => { expect(secondOwner.startsWith('worker:')).toBe(true); expect(firstOwner).not.toEqual(secondOwner); }); + + describe('lease duration', () => { + // The expiry decides how long a job stays blocked after a process dies without releasing. + // Reading it off the job's own timeout made that window as long as the job was allowed to + // take — up to two hours for the jobs declaring the longest timeouts. + + it('claims for a minute, whatever the job it guards is allowed to take', async () => { + const { service, onQuery } = buildService({}); + + await service.acquire('SomeService::job'); + + expect(onQuery.mock.calls[0][1][2]).toEqual('60'); + }); + + it('renews for the same short span', async () => { + const { service, onQuery } = buildService({}); + + await service.renew('SomeService::job'); + + expect(onQuery.mock.calls[0][1][2]).toEqual('60'); + }); + }); + + describe('shutdown', () => { + // A deployment sends SIGTERM in the middle of a run. Whatever happens here decides whether the + // successor can pick the job up, and whether it can pick it up while this process still works + // on it. + + /** Lets pending promises settle without advancing any timer. */ + const settle = () => new Promise((resolve) => setImmediate(resolve)); + + const released = (onQuery: jest.Mock) => + onQuery.mock.calls.some(([sql]) => (sql as string).includes('DELETE FROM')); + + it('does not take the lease away from a job that is still running', async () => { + // The dangerous direction. Releasing on SIGTERM would let the successor claim the lease and + // start the same job while this process keeps working on it for the rest of its stop grace + // period — the double run the lease exists to prevent. + jest.useFakeTimers({ doNotFake: ['setImmediate', 'nextTick'] }); + + try { + const { service, onQuery } = buildService({}); + let finish: () => void; + const run = service.run('SomeService::job', () => new Promise((resolve) => (finish = resolve))); + + await settle(); + + const shutdown = service.beforeApplicationShutdown(); + await settle(); + + // Grace expires with the job still working. + jest.advanceTimersByTime(10_000); + await shutdown; + + expect(released(onQuery)).toBe(false); + + finish(); + await run; + } finally { + jest.useRealTimers(); + } + }); + + it('waits for the running job instead of letting the process leave under it', async () => { + // The reason for waiting at all: the run gets to reach its own release, so the successor + // finds no row and starts the job on its next tick instead of sitting out the expiry. + // Without the hook the shutdown would be over before the job is, which is what the pending + // assertion below pins — the release alone proves nothing, it happens either way. + const { service, onQuery } = buildService({}); + let finish: () => void; + const run = service.run('SomeService::job', () => new Promise((resolve) => (finish = resolve))); + + await settle(); + + let over = false; + const shutdown = service.beforeApplicationShutdown().then(() => (over = true)); + await settle(); + + expect(over).toBe(false); + expect(released(onQuery)).toBe(false); + + finish(); + await run; + await shutdown; + + expect(over).toBe(true); + expect(released(onQuery)).toBe(true); + }); + + it('is reached at all, because bootstrap enables the hooks', () => { + // Nest calls no shutdown hook unless the application asks for it. Without that one line + // everything above is dead code and a deployment kills the process mid-job, which is the + // state this change came from. Read from the source because there is nothing to call. + const main = readFileSync(join(__dirname, '..', '..', '..', 'main.ts'), 'utf8'); + + expect(main).toMatch(/app\.enableShutdownHooks\(/); + expect(main).toContain("'SIGTERM'"); + }); + + it('returns immediately when no job is running', async () => { + // The overwhelmingly common case on a deployment: nothing outstanding, so shutdown must not + // spend the grace period waiting for it. + const { service } = buildService({}); + + await service.beforeApplicationShutdown(); + }); + + it('stops tracking a run once it is done, so a later shutdown has nothing to wait for', async () => { + const { service } = buildService({}); + + await service.run('SomeService::job', jest.fn().mockResolvedValue(undefined)); + + expect([...(service['inFlight'] as Map).keys()]).toEqual([]); + }); + }); }); diff --git a/src/shared/services/__tests__/dfx-cron.service.spec.ts b/src/shared/services/__tests__/dfx-cron.service.spec.ts index 98c438879b..fb35512f88 100644 --- a/src/shared/services/__tests__/dfx-cron.service.spec.ts +++ b/src/shared/services/__tests__/dfx-cron.service.spec.ts @@ -10,6 +10,7 @@ import { CronExpression, SchedulerRegistry } from '@nestjs/schedule'; import { ConfigService, GetConfig } from 'src/config/config'; import { CronScope, DFX_CRONJOB_PARAMS, DfxCronParams } from 'src/shared/utils/cron'; import { CronJob } from 'cron'; +import { DataSource } from 'typeorm'; import { CronLeaseService } from '../cron-lease.service'; import { DfxCronService } from '../dfx-cron.service'; import { Process } from '../process.service'; @@ -60,7 +61,7 @@ function buildService(providers: { instance: object }[]): { // Runs the task straight through: these tests are about which jobs get registered, not // about the lease. What the lease itself does has its own suite. const leases = createMock({ - run: (_job: string, _ttl: number, task: () => Promise) => task(), + run: (_job: string, task: () => Promise) => task(), }); return { service: new DfxCronService(discovery, metadataScanner, scheduler, leases), scheduler }; @@ -190,7 +191,7 @@ describe('DfxCronService', () => { }); const metadataScanner = createMock({ getAllMethodNames: (i: object) => Object.keys(i) }); const leaseSpy = createMock({ - run: (job: string, _ttl: number, task: () => Promise) => { + run: (job: string, task: () => Promise) => { seen.push(job); return task(); }, @@ -222,6 +223,48 @@ describe('DfxCronService', () => { expect(leased).not.toContain('Object::bothJob'); }); + + it('does not turn a long job timeout into a long lease', async () => { + // The lease used to expire when the job's own timeout did. Sixteen jobs declare 7200 — + // seconds, per LockClass — so a process killed mid-run left the row behind for two hours + // and its successor sat the job out for that long, silently. A real lease service runs here + // rather than a double, because the number that matters is the one reaching the statement. + process.env.CRON_ROLE = 'worker'; + new ConfigService(GetConfig()); + + const query = jest.fn().mockImplementation((sql: string) => { + if (sql.includes('INSERT INTO')) return Promise.resolve([{ owner: 'worker:1' }]); + if (sql.includes('UPDATE')) return Promise.resolve([[], 1]); + return Promise.resolve([]); + }); + const leaseService = new CronLeaseService(createMock({ query })); + + const jobs = [ + providerWithJob('longRunningJob', { + expression: CronExpression.EVERY_HOUR, + scope: CronScope.WORKER, + useDelay: false, + timeout: 7200, + }), + ]; + const discovery = createMock({ + getProviders: () => + jobs.map((p) => ({ ...p, isDependencyTreeStatic: () => true })) as ReturnType< + DiscoveryService['getProviders'] + >, + }); + const metadataScanner = createMock({ getAllMethodNames: (i: object) => Object.keys(i) }); + + new DfxCronService(discovery, metadataScanner, createMock(), leaseService).onModuleInit(); + + const scheduled = (CronJob as unknown as jest.Mock).mock.calls.map(([, fn]) => fn as () => unknown); + for (const fire of scheduled) await fire(); + + const claim = query.mock.calls.find(([sql]) => (sql as string).includes('INSERT INTO')); + + expect(claim).toBeDefined(); + expect(claim[1][2]).toEqual('60'); + }); }); describe('role heartbeat', () => { diff --git a/src/shared/services/cron-lease.service.ts b/src/shared/services/cron-lease.service.ts index 53a5f281ce..c7b1e87b6b 100644 --- a/src/shared/services/cron-lease.service.ts +++ b/src/shared/services/cron-lease.service.ts @@ -1,9 +1,40 @@ -import { Injectable } from '@nestjs/common'; +import { BeforeApplicationShutdown, Injectable } from '@nestjs/common'; import { randomUUID } from 'crypto'; import { Config } from 'src/config/config'; import { DataSource } from 'typeorm'; import { DfxLogger } from './dfx-logger'; +/** + * How long a claim stays valid without being renewed. + * + * Deliberately short, and deliberately unrelated to how long the job it guards may run. A lease + * expiry is not a job timeout: its only purpose is to bound how long a claim outlives a process + * that can no longer speak for itself — SIGKILL, an OOM kill, a lost machine. In each of those the + * row stays behind until it expires, and for that window the job runs nowhere. + * + * Deriving the expiry from the job's own timeout got that backwards. A timeout answers "how long + * may this run take", which says nothing about how long a stale claim should survive its owner, + * and it made the outage longest for exactly the jobs that declare the longest timeouts. One + * minute is long enough for the renewal below to carry a healthy run across a slow query or a + * brief connection hiccup, and short enough that the worst case is a minute of one job not + * running. + */ +const LEASE_TTL_SECONDS = 60; + +/** + * Renew at a third of the lease, so two consecutive failed renewals still leave a full attempt + * before the claim lapses. + */ +const RENEWAL_INTERVAL_MS = (LEASE_TTL_SECONDS / 3) * 1000; + +/** + * How long shutdown waits for jobs that are still running. See `beforeApplicationShutdown`. + * + * Short on purpose: it is a handover courtesy, not a completion guarantee. Every process pays it + * on every deployment, and the container's own stop grace period ends the process regardless. + */ +const SHUTDOWN_GRACE_MS = 10 * 1000; + /** * A lease on a scheduled job, held in the database for as long as the job runs. * @@ -23,16 +54,25 @@ import { DfxLogger } from './dfx-logger'; * be extended and eventually expires — a second process may then start the same job while the * first is still working. Preventing that outright would require every write inside every job to * carry the lease token, which this does not attempt. What it does is turn an unbounded window - * ("until a human reads the alert") into a bounded one (the lease duration). A lost lease is + * ("until a human reads the alert") into a bounded one (`LEASE_TTL_SECONDS`). A lost lease is * logged at error level, because it means the run that is still going has no claim to the job any * more. */ @Injectable() -export class CronLeaseService { +export class CronLeaseService implements BeforeApplicationShutdown { private readonly logger = new DfxLogger(CronLeaseService); private ownerId?: string; + /** + * The runs this process currently holds a lease for, by job name. + * + * Kept so shutdown knows what is still outstanding. The stored promise has its rejection already + * absorbed: the job's own error belongs to its caller, and a second consumer of the same + * rejection here would surface as an unhandled one. + */ + private readonly inFlight = new Map>(); + /** * Identifies THIS process for the lifetime of the process. The role makes a stray row readable * for an operator; the random part is what actually distinguishes two processes of the same @@ -55,7 +95,7 @@ export class CronLeaseService { * them sees a returned row. Read the `WHERE` as "only if nobody is currently holding it" — an * unexpired row belonging to another process leaves the update out and returns nothing. */ - async acquire(job: string, ttlSeconds: number): Promise { + async acquire(job: string): Promise { const claimed = await this.dataSource.query( `INSERT INTO "cron_lease" ("name", "owner", "acquired", "expires") VALUES ($1, $2, now(), now() + ($3 || ' seconds')::interval) @@ -63,7 +103,7 @@ export class CronLeaseService { SET "owner" = EXCLUDED."owner", "acquired" = EXCLUDED."acquired", "expires" = EXCLUDED."expires" WHERE "cron_lease"."expires" <= now() RETURNING "owner"`, - [job, this.owner, `${ttlSeconds}`], + [job, this.owner, `${LEASE_TTL_SECONDS}`], ); return claimed.length > 0; @@ -74,12 +114,12 @@ export class CronLeaseService { * longer the owner — which means another process has taken the job over and this run should be * treated as having lost its claim. */ - async renew(job: string, ttlSeconds: number): Promise { + async renew(job: string): Promise { const [, affected] = await this.dataSource.query( `UPDATE "cron_lease" SET "expires" = now() + ($3 || ' seconds')::interval WHERE "name" = $1 AND "owner" = $2`, - [job, this.owner, `${ttlSeconds}`], + [job, this.owner, `${LEASE_TTL_SECONDS}`], ); return affected > 0; @@ -100,10 +140,10 @@ export class CronLeaseService { * the assumption that it is probably alone. The caller sees the same outcome as a job whose * lease is held elsewhere — it simply does not run this cycle and tries again on the next. */ - async run(job: string, ttlSeconds: number, task: () => Promise): Promise { + async run(job: string, task: () => Promise): Promise { let acquired: boolean; try { - acquired = await this.acquire(job, ttlSeconds); + acquired = await this.acquire(job); } catch (e) { this.logger.error(`Skipping ${job}: could not reach the lease table`, e); return; @@ -111,25 +151,69 @@ export class CronLeaseService { if (!acquired) return; - // Renew at a third of the lease so two consecutive failures still leave a full attempt before - // the lease lapses. Unref'd: a pending timer must never hold the process open on shutdown. - const renewal = setInterval( - () => { - void this.renew(job, ttlSeconds) - .then((stillOurs) => { - if (!stillOurs) this.logger.error(`Lost the lease for ${job} while it was still running`); - }) - .catch((e) => this.logger.error(`Could not extend the lease for ${job}`, e)); - }, - (ttlSeconds / 3) * 1000, - ); + // Unref'd: a pending timer must never hold the process open on shutdown. + const renewal = setInterval(() => { + void this.renew(job) + .then((stillOurs) => { + if (!stillOurs) this.logger.error(`Lost the lease for ${job} while it was still running`); + }) + .catch((e) => this.logger.error(`Could not extend the lease for ${job}`, e)); + }, RENEWAL_INTERVAL_MS); renewal.unref(); - try { - await task(); - } finally { - clearInterval(renewal); - await this.release(job).catch((e) => this.logger.error(`Could not release the lease for ${job}`, e)); - } + const run = (async () => { + try { + await task(); + } finally { + clearInterval(renewal); + await this.release(job).catch((e) => this.logger.error(`Could not release the lease for ${job}`, e)); + this.inFlight.delete(job); + } + })(); + + this.inFlight.set( + job, + run.catch(() => undefined), + ); + + return run; + } + + /** + * Waits for the jobs this process is still running, so their normal release path can hand the + * lease over to the successor instead of leaving it to expire. + * + * `beforeApplicationShutdown`, not `onApplicationShutdown`: TypeOrmCoreModule closes the + * connection in the latter, and releasing a lease needs that connection. + * + * A lease is NOT taken away from a job that is still working. Releasing on SIGTERM would hand + * over faster, but the job keeps running until the container's stop grace period ends it — + * `dfx-api-worker` is configured to allow two minutes — and a successor claiming the freed lease + * inside that window would run the same money-moving job alongside it. That is the outcome this + * whole mechanism exists to prevent, so it is not traded for a faster handover. + * + * What is still running after the wait therefore keeps its lease, which lapses within + * `LEASE_TTL_SECONDS` of the last renewal. The renewal timers deliberately keep going meanwhile: + * they hold the claim for as long as this process is alive to renew it. + */ + async beforeApplicationShutdown(): Promise { + const running = [...this.inFlight.values()]; + if (!running.length) return; + + this.logger.info(`Shutting down: waiting up to ${SHUTDOWN_GRACE_MS / 1000}s for ${running.length} running job(s)`); + + await Promise.race([Promise.all(running), this.shutdownGrace()]); + + const stranded = [...this.inFlight.keys()]; + if (stranded.length) + this.logger.warn( + `Shutting down with ${stranded.length} job(s) still running (${stranded.join(', ')}); ` + + `their leases stay held and lapse within ${LEASE_TTL_SECONDS}s`, + ); + } + + private shutdownGrace(): Promise { + // Unref'd so winning the race above does not keep the process alive for the rest of the grace. + return new Promise((resolve) => setTimeout(resolve, SHUTDOWN_GRACE_MS).unref()); } } diff --git a/src/shared/services/dfx-cron.service.ts b/src/shared/services/dfx-cron.service.ts index 31e4a9eaa2..e01d6eab61 100644 --- a/src/shared/services/dfx-cron.service.ts +++ b/src/shared/services/dfx-cron.service.ts @@ -11,9 +11,6 @@ import { Util } from 'src/shared/utils/util'; import { CustomCronExpression } from '../utils/custom-cron-expression'; import { DfxLogger } from './dfx-logger'; -/** Lease length for a job that declares no timeout of its own. */ -const DEFAULT_LEASE_SECONDS = 300; - interface CronJobData { instance: object; methodRef: any; @@ -156,12 +153,7 @@ export class DfxCronService implements OnModuleInit { if (data.params.scope === CronScope.BOTH) return task; - // The lease has to outlive a single run, so it follows the job’s own timeout where there is - // one. Where there is none the job carries no expectation of its duration either, and five - // minutes is long enough that the renewal (every third of it) keeps a healthy run alive. - const ttlSeconds = Number.isFinite(data.params.timeout) ? data.params.timeout : DEFAULT_LEASE_SECONDS; - - return () => this.leases.run(cronJobName, ttlSeconds, task); + return () => this.leases.run(cronJobName, task); } private wrapFunction(data: CronJobData) { From 3602fc9ec4a2fdafbf21a0c73b2c9cbbd79a23c1 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:30:59 +0200 Subject: [PATCH 36/86] Stop a broken lease table from reading as a healthy process MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without the table — a process started ahead of the migration, a revoked grant, a database that is briefly gone — every claim fails and every worker- and api-scoped job is skipped on every tick. Skipping is the right answer, and CONTRIBUTING asks for exactly it: a job that moves money must not proceed on the assumption that it is probably alone. What was missing is that nobody could tell. The skip looks like a job that had nothing to do, and the role heartbeat is scope `both`, so the lease never touches it — it kept reporting the same healthy line while everything it counts sat out. It counts REGISTERED jobs, which cannot see this at all. The lease layer now carries its own state: a read of the table at start-up, a failure count, and a health flag that stays false until an operation gets through, so a role whose jobs are all sitting out keeps reporting rather than falling quiet after one window. The heartbeat reads it and, when it is bad, writes the same line at error level with the reason appended — the same line because the role alert matches on its shape, and a heartbeat that stopped matching would read as a dead process and hide the cause instead of naming it. Deliberately not a boot crash: the restart loop would be self-inflicted during the very rollout that adds the table, against a database that is correct a minute later. --- .../__tests__/cron-lease.service.spec.ts | 52 ++++++++++++ .../__tests__/dfx-cron.service.spec.ts | 35 ++++++++ src/shared/services/cron-lease.service.ts | 79 ++++++++++++++++++- src/shared/services/dfx-cron.service.ts | 16 +++- 4 files changed, 177 insertions(+), 5 deletions(-) diff --git a/src/shared/services/__tests__/cron-lease.service.spec.ts b/src/shared/services/__tests__/cron-lease.service.spec.ts index 46a3b695bd..422a28addc 100644 --- a/src/shared/services/__tests__/cron-lease.service.spec.ts +++ b/src/shared/services/__tests__/cron-lease.service.spec.ts @@ -244,4 +244,56 @@ describe('CronLeaseService', () => { expect([...(service['inFlight'] as Map).keys()]).toEqual([]); }); }); + + describe('visibility of an unusable lease table', () => { + // Without the table every claim fails and every worker- and api-scoped job is skipped. Not + // running is the right answer; looking healthy while doing it is not, and the role heartbeat + // is exempt from the lease so it never noticed. + + it('says so at start-up when the table cannot be read', async () => { + const onQuery = jest.fn().mockRejectedValue(new Error('relation "cron_lease" does not exist')); + const { service } = buildService({ onQuery }); + + await service.onModuleInit(); + + expect(service.takeFailures()).toEqual({ + healthy: false, + count: 1, + last: 'relation "cron_lease" does not exist', + }); + }); + + it('stays unusable across heartbeats until an operation succeeds', async () => { + // A role whose jobs are all sitting out produces no new failures either. Were the state + // per window, the second heartbeat would look healthy again while nothing had changed. + const onQuery = jest.fn().mockRejectedValue(new Error('connection refused')); + const { service } = buildService({ onQuery }); + + await service.run('SomeService::job', jest.fn()); + + expect(service.takeFailures()).toMatchObject({ healthy: false, count: 1 }); + expect(service.takeFailures()).toMatchObject({ healthy: false, count: 0 }); + }); + + it('reports healthy again once a claim gets through', async () => { + // The counterpart: a transient outage must not leave the process reporting an error for the + // rest of its life. + const onQuery = jest.fn().mockRejectedValueOnce(new Error('connection refused')); + const { service } = buildService({ onQuery }); + + await service.onModuleInit(); + expect(service.takeFailures().healthy).toBe(false); + + onQuery.mockResolvedValue([{ owner: 'worker:1' }]); + await service.acquire('SomeService::job'); + + expect(service.takeFailures().healthy).toBe(true); + }); + + it('starts out healthy, so the heartbeat does not cry wolf before anything ran', () => { + const { service } = buildService({}); + + expect(service.takeFailures()).toEqual({ healthy: true, count: 0, last: undefined }); + }); + }); }); diff --git a/src/shared/services/__tests__/dfx-cron.service.spec.ts b/src/shared/services/__tests__/dfx-cron.service.spec.ts index fb35512f88..e1cd751254 100644 --- a/src/shared/services/__tests__/dfx-cron.service.spec.ts +++ b/src/shared/services/__tests__/dfx-cron.service.spec.ts @@ -62,6 +62,7 @@ function buildService(providers: { instance: object }[]): { // about the lease. What the lease itself does has its own suite. const leases = createMock({ run: (_job: string, task: () => Promise) => task(), + takeFailures: () => ({ healthy: true, count: 0 }), }); return { service: new DfxCronService(discovery, metadataScanner, scheduler, leases), scheduler }; @@ -195,6 +196,7 @@ describe('DfxCronService', () => { seen.push(job); return task(); }, + takeFailures: () => ({ healthy: true, count: 0 }), }); const registry = createMock(); @@ -307,6 +309,39 @@ describe('DfxCronService', () => { expect(info).toHaveBeenCalledWith('CronRole worker: heartbeat, 4 jobs registered'); }); + it('reports an unusable lease instead of the healthy line', () => { + // The state this exists for: without the table every worker- and api-scoped job is skipped + // on every tick, and nothing said so. This job is scope `both`, so the lease never touches + // it — it kept reporting a healthy process while everything it counts sat out. The count is + // of REGISTERED jobs and cannot see it either. + process.env.CRON_ROLE = 'worker'; + new ConfigService(GetConfig()); + + const unhealthy = createMock({ + run: (_job: string, task: () => Promise) => task(), + takeFailures: () => ({ healthy: false, count: 3, last: 'relation "cron_lease" does not exist' }), + }); + const discovery = createMock({ getProviders: () => [] }); + const metadataScanner = createMock({ getAllMethodNames: () => [] }); + const service = new DfxCronService(discovery, metadataScanner, createMock(), unhealthy); + + const error = jest.spyOn(service['logger'], 'error'); + const info = jest.spyOn(service['logger'], 'info'); + + service.reportRole(); + + expect(info).not.toHaveBeenCalled(); + expect(error).toHaveBeenCalledTimes(1); + + const line = error.mock.calls[0][0] as string; + + // Still the shape the role alert matches — a heartbeat that stops matching would read as a + // dead process and hide the reason rather than name it. + expect(line).toMatch(/CronRole (api|worker|all): heartbeat, [0-9]+ jobs registered/); + expect(line).toContain('lease unusable'); + expect(line).toContain('relation "cron_lease" does not exist'); + }); + it('produces a line the alert query actually matches', () => { // The alert reads this line with `CronRole (api|worker|all): heartbeat, [0-9]+ jobs // registered`. Pinning the wording here is the only place that couples the two: nothing in diff --git a/src/shared/services/cron-lease.service.ts b/src/shared/services/cron-lease.service.ts index c7b1e87b6b..a2eee003ed 100644 --- a/src/shared/services/cron-lease.service.ts +++ b/src/shared/services/cron-lease.service.ts @@ -1,4 +1,4 @@ -import { BeforeApplicationShutdown, Injectable } from '@nestjs/common'; +import { BeforeApplicationShutdown, Injectable, OnModuleInit } from '@nestjs/common'; import { randomUUID } from 'crypto'; import { Config } from 'src/config/config'; import { DataSource } from 'typeorm'; @@ -59,7 +59,7 @@ const SHUTDOWN_GRACE_MS = 10 * 1000; * more. */ @Injectable() -export class CronLeaseService implements BeforeApplicationShutdown { +export class CronLeaseService implements OnModuleInit, BeforeApplicationShutdown { private readonly logger = new DfxLogger(CronLeaseService); private ownerId?: string; @@ -73,6 +73,16 @@ export class CronLeaseService implements BeforeApplicationShutdown { */ private readonly inFlight = new Map>(); + /** + * Whether the last lease operation reached the table. Sticky until one succeeds, so a role whose + * jobs all sit out still reports the state rather than only the tick that first hit it. + */ + private healthy = true; + + /** Lease operations that failed since the role heartbeat last read them; see `takeFailures`. */ + private failures = 0; + private lastFailure?: string; + /** * Identifies THIS process for the lifetime of the process. The role makes a stray row readable * for an operator; the random part is what actually distinguishes two processes of the same @@ -87,6 +97,35 @@ export class CronLeaseService implements BeforeApplicationShutdown { constructor(private readonly dataSource: DataSource) {} + /** + * Reads the lease table once, so a process that cannot use it says so at start-up. + * + * Without the table — a process started before the migration ran, a revoked grant, a database + * that is not there yet — every worker- and api-scoped job fails its claim and is skipped. That + * is the correct behaviour, and CONTRIBUTING asks for exactly it where the alternative is + * proceeding on an unverified assumption. The problem it left behind is a reporting one: the + * skip is indistinguishable from a job that had nothing to do, and the role heartbeat is scoped + * `both`, so it is exempt from the lease and keeps reporting a healthy process. + * + * This does not take the boot down. A crash loop here would be loud, but it would also be + * self-inflicted during the very rollout that introduces the table: the migration ships with the + * process that runs migrations, and the other one would restart against a database that is + * correct a minute later. Reporting is what was missing, so reporting is what this adds — here, + * and continuously through `takeFailures`, because a boot line scrolls out of an alert window + * and says nothing about a table that disappeared afterwards. + */ + async onModuleInit(): Promise { + try { + await this.dataSource.query(`SELECT 1 FROM "cron_lease" LIMIT 1`); + } catch (e) { + this.recordFailure(e); + this.logger.error( + 'The cron lease table cannot be read: every worker- and api-scoped job will be skipped on every tick', + e, + ); + } + } + /** * Claims the lease for `job`, or reports that someone else holds it. * @@ -106,6 +145,8 @@ export class CronLeaseService implements BeforeApplicationShutdown { [job, this.owner, `${LEASE_TTL_SECONDS}`], ); + this.healthy = true; + return claimed.length > 0; } @@ -145,6 +186,7 @@ export class CronLeaseService implements BeforeApplicationShutdown { try { acquired = await this.acquire(job); } catch (e) { + this.recordFailure(e); this.logger.error(`Skipping ${job}: could not reach the lease table`, e); return; } @@ -157,7 +199,10 @@ export class CronLeaseService implements BeforeApplicationShutdown { .then((stillOurs) => { if (!stillOurs) this.logger.error(`Lost the lease for ${job} while it was still running`); }) - .catch((e) => this.logger.error(`Could not extend the lease for ${job}`, e)); + .catch((e) => { + this.recordFailure(e); + this.logger.error(`Could not extend the lease for ${job}`, e); + }); }, RENEWAL_INTERVAL_MS); renewal.unref(); @@ -166,7 +211,10 @@ export class CronLeaseService implements BeforeApplicationShutdown { await task(); } finally { clearInterval(renewal); - await this.release(job).catch((e) => this.logger.error(`Could not release the lease for ${job}`, e)); + await this.release(job).catch((e) => { + this.recordFailure(e); + this.logger.error(`Could not release the lease for ${job}`, e); + }); this.inFlight.delete(job); } })(); @@ -212,6 +260,29 @@ export class CronLeaseService implements BeforeApplicationShutdown { ); } + /** + * The state of the lease layer, for the role heartbeat to report. + * + * Read rather than pushed: a lease that cannot reach its table stops every worker- and + * api-scoped job, and no other line says so — the jobs simply do not run. `healthy` stays false + * until an operation succeeds, so a role whose jobs are all sitting out keeps reporting it + * instead of falling quiet after the first window. The counter is per window; the last message + * is not, so an unhealthy report always names something. + */ + takeFailures(): { healthy: boolean; count: number; last?: string } { + const taken = { healthy: this.healthy, count: this.failures, last: this.lastFailure }; + + this.failures = 0; + + return taken; + } + + private recordFailure(e: unknown): void { + this.failures++; + this.lastFailure = e instanceof Error ? e.message : String(e); + this.healthy = false; + } + private shutdownGrace(): Promise { // Unref'd so winning the race above does not keep the process alive for the rest of the grace. return new Promise((resolve) => setTimeout(resolve, SHUTDOWN_GRACE_MS).unref()); diff --git a/src/shared/services/dfx-cron.service.ts b/src/shared/services/dfx-cron.service.ts index e01d6eab61..13ac2a35d6 100644 --- a/src/shared/services/dfx-cron.service.ts +++ b/src/shared/services/dfx-cron.service.ts @@ -98,7 +98,21 @@ export class DfxCronService implements OnModuleInit { // for spreading load. @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.BOTH, useDelay: false }) reportRole(): void { - this.logger.info(`CronRole ${Config.cronRole}: heartbeat, ${this.registeredCount} jobs registered`); + const line = `CronRole ${Config.cronRole}: heartbeat, ${this.registeredCount} jobs registered`; + const lease = this.leases.takeFailures(); + + if (lease.healthy) return this.logger.info(line); + + // A job that cannot take its lease does not run, and nothing else says so — the skip looks + // exactly like a job with nothing to do. This job is scope `both` and therefore exempt from + // the lease itself, so it keeps reporting while everything it counts is sitting out: a count + // of REGISTERED jobs cannot see that. Same line, because the role alert matches on its shape; + // the state is appended and the level raised. + this.logger.error( + `${line}, lease unusable: ${lease.count} failure(s) since the last heartbeat, last error: ${ + lease.last ?? 'unknown' + }`, + ); } /** From e1e38d58c9cb2e69e37b9b67eaa5d6aa2566ffa5 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:00 +0200 Subject: [PATCH 37/86] Name the lease primary key the way TypeORM names one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CONTRIBUTING.md requires hand-written schema migrations to match TypeORM's deterministic constraint naming; `PK_cron_lease_name` was assembled out of words instead. A constraint TypeORM does not recognise as its own is one it offers to create — a schema comparison reports it missing and a generated migration or a synchronize run acts on that, against the table the job lock lives in. The name is `PK_` plus the first 27 characters of sha1('cron_lease_name'), which is where PK_a12c181c2b26f33be13d55a15af comes from. Edited in place rather than followed up, because this migration exists only on this branch and CONTRIBUTING forbids editing one after it reaches DEV, not before. A guard comes with it, since the rule had nothing enforcing it: it recomputes every primary key declared in a CREATE TABLE across all migrations — 102 of them, all matching — and checks that no constraint of any kind carries a spelled-out name. Both flag this one when the old name is put back. --- migration/1785600000000-AddCronLease.js | 6 +- .../migration-constraint-naming.spec.ts | 84 +++++++++++++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 src/__tests__/migration-constraint-naming.spec.ts diff --git a/migration/1785600000000-AddCronLease.js b/migration/1785600000000-AddCronLease.js index 65064be081..4fc252f879 100644 --- a/migration/1785600000000-AddCronLease.js +++ b/migration/1785600000000-AddCronLease.js @@ -20,6 +20,10 @@ * tables without a primary key from the MSSQL cutover — a FK into one of them would fail at boot. * `name` is the primary key, so the claim is a single atomic upsert with no index to keep in sync. * + * The primary key carries the name TypeORM derives for it, per the rule in CONTRIBUTING.md: `PK_` + * plus the first 27 characters of sha1('cron_lease_name'). A hand-picked name would not be + * recognised as its own by a schema comparison, which would offer to drop and recreate it. + * * @class @implements {MigrationInterface} */ module.exports = class AddCronLease1785600000000 { @@ -30,7 +34,7 @@ module.exports = class AddCronLease1785600000000 { */ async up(queryRunner) { await queryRunner.query( - `CREATE TABLE "cron_lease" ("name" character varying(256) NOT NULL, "owner" character varying(256) NOT NULL, "acquired" TIMESTAMP NOT NULL DEFAULT now(), "expires" TIMESTAMP NOT NULL, CONSTRAINT "PK_cron_lease_name" PRIMARY KEY ("name"))`, + `CREATE TABLE "cron_lease" ("name" character varying(256) NOT NULL, "owner" character varying(256) NOT NULL, "acquired" TIMESTAMP NOT NULL DEFAULT now(), "expires" TIMESTAMP NOT NULL, CONSTRAINT "PK_a12c181c2b26f33be13d55a15af" PRIMARY KEY ("name"))`, ); } diff --git a/src/__tests__/migration-constraint-naming.spec.ts b/src/__tests__/migration-constraint-naming.spec.ts new file mode 100644 index 0000000000..a4518326e1 --- /dev/null +++ b/src/__tests__/migration-constraint-naming.spec.ts @@ -0,0 +1,84 @@ +import { createHash } from 'crypto'; +import { readFileSync, readdirSync } from 'fs'; +import { join } from 'path'; + +/** + * Hand-written schema migrations must name their constraints the way TypeORM does. CONTRIBUTING.md + * states the rule and the algorithm; this checks that the files obey it. + * + * It is not a style question. A constraint TypeORM does not recognise as its own is one it offers + * to create — a schema comparison sees a name it would never have produced, reports the constraint + * as missing, and a generated migration or a `synchronize` run acts on that. + * + * Two checks, deliberately: the recomputation covers the primary keys declared inside a + * `CREATE TABLE`, where table and columns are both in front of us, and the shape check covers + * every other constraint kind, where they are not. The shape check is the weaker statement — a + * hexadecimal name can still be the wrong hexadecimal name — but it catches the case this guard + * was written for, a name assembled out of words. + */ +const MIGRATIONS = join(__dirname, '..', '..', 'migration'); + +/** From CONTRIBUTING.md: `_ + sha1(tableName + '_' + columnNames.sort().join('_'))`. */ +function typeormName(prefix: string, length: number, table: string, columns: string[]): string { + return `${prefix}_${createHash('sha1') + .update(`${table}_${[...columns].sort().join('_')}`) + .digest('hex') + .substring(0, length)}`; +} + +interface Migration { + file: string; + content: string; +} + +describe('migration constraint naming', () => { + const migrations: Migration[] = readdirSync(MIGRATIONS) + .filter((file) => file.endsWith('.js')) + .map((file) => ({ file, content: readFileSync(join(MIGRATIONS, file), 'utf8') })); + + it('finds the migrations to check', () => { + // Guards against the whole suite passing because the directory was read from the wrong place. + expect(migrations.length).toBeGreaterThan(50); + }); + + describe('primary keys declared in a CREATE TABLE', () => { + const declared = migrations.flatMap(({ file, content }) => + [...content.matchAll(/CREATE TABLE "(\w+)" \((?:.*?)CONSTRAINT "(PK_\w+)" PRIMARY KEY \(([^)]*)\)/gs)].map( + ([, table, name, columns]) => ({ + file, + table, + name, + columns: columns.split(',').map((column) => column.trim().replace(/"/g, '')), + }), + ), + ); + + it('finds primary keys to check', () => { + expect(declared.length).toBeGreaterThan(50); + }); + + it('names every one of them the way TypeORM would', () => { + const wrong = declared + .filter(({ table, name, columns }) => name !== typeormName('PK', 27, table, columns)) + .map(({ file, table, name, columns }) => ({ + file, + name, + expected: typeormName('PK', 27, table, columns), + })); + + expect(wrong).toEqual([]); + }); + }); + + it('gives every constraint a hashed name rather than a spelled-out one', () => { + // The shape the algorithm produces: a prefix and hexadecimal. A name built from the table and + // column it belongs to reads correctly and is exactly the case this catches. + const spelledOut = migrations.flatMap(({ file, content }) => + [...content.matchAll(/CONSTRAINT "((?:PK|FK|UQ|DF|REL|IDX|CHK)_\w+)"/g)] + .map(([, name]) => ({ file, name })) + .filter(({ name }) => !/^(?:PK|FK|UQ|DF|REL|IDX|CHK)_[a-f0-9]+$/.test(name)), + ); + + expect(spelledOut).toEqual([]); + }); +}); From 8e45d4d8768afdf609e815f4e97021941dbdfa5f Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:01 +0200 Subject: [PATCH 38/86] Give the lease table an entity, and its timestamps a time zone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two problems in one table definition. The table existed only as DDL inside a migration, so the entity model does not know about it. The next `npm run migration` compares the two and reads the absence as an instruction: the generated migration would carry a `DROP TABLE "cron_lease"`, and the job lock would disappear with nobody deciding it should. The entity now mirrors the DDL — it stays there for the schema comparison, not to be read through, since the claim is a single `INSERT .. ON CONFLICT .. WHERE` the query builder cannot express. `acquired` and `expires` were `TIMESTAMP` without a zone while the statements compare them against `now()`, which has one. That comparison then runs through whatever time zone the session carries: the same row expires an hour late or an hour early across a daylight saving change, and inconsistently between two sessions that disagree. An hour late is a job that runs nowhere, an hour early is two processes running it at once — both of the failures this table exists to rule out. Both columns are `timestamptz` now, in the DDL and in the entity. The test builds the entity metadata without a connection and asks TypeORM's own driver and naming strategy what it would emit, then checks that against the migration file. It fails on a type change on either side and on a column present in only one of them. --- migration/1785600000000-AddCronLease.js | 12 ++- .../__tests__/cron-lease.entity.spec.ts | 90 +++++++++++++++++++ .../models/cron-lease/cron-lease.entity.ts | 39 ++++++++ src/shared/shared.module.ts | 6 +- 4 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 src/shared/models/cron-lease/__tests__/cron-lease.entity.spec.ts create mode 100644 src/shared/models/cron-lease/cron-lease.entity.ts diff --git a/migration/1785600000000-AddCronLease.js b/migration/1785600000000-AddCronLease.js index 4fc252f879..82b2736a4f 100644 --- a/migration/1785600000000-AddCronLease.js +++ b/migration/1785600000000-AddCronLease.js @@ -24,6 +24,16 @@ * plus the first 27 characters of sha1('cron_lease_name'). A hand-picked name would not be * recognised as its own by a schema comparison, which would offer to drop and recreate it. * + * Both timestamps carry their time zone. They are compared against `now()` inside the claim and + * renewal statements, and a value without a zone on one side of that comparison is resolved + * through whatever time zone the session carries — which makes the same row expire an hour late + * or an hour early across a daylight saving change, and inconsistently between two sessions that + * disagree. An hour late is a job that runs nowhere, an hour early is two processes running it at + * once. + * + * The shape of the table is mirrored by src/shared/models/cron-lease/cron-lease.entity.ts, without + * which a generated migration would read the table as one to drop. + * * @class @implements {MigrationInterface} */ module.exports = class AddCronLease1785600000000 { @@ -34,7 +44,7 @@ module.exports = class AddCronLease1785600000000 { */ async up(queryRunner) { await queryRunner.query( - `CREATE TABLE "cron_lease" ("name" character varying(256) NOT NULL, "owner" character varying(256) NOT NULL, "acquired" TIMESTAMP NOT NULL DEFAULT now(), "expires" TIMESTAMP NOT NULL, CONSTRAINT "PK_a12c181c2b26f33be13d55a15af" PRIMARY KEY ("name"))`, + `CREATE TABLE "cron_lease" ("name" character varying(256) NOT NULL, "owner" character varying(256) NOT NULL, "acquired" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "expires" TIMESTAMP WITH TIME ZONE NOT NULL, CONSTRAINT "PK_a12c181c2b26f33be13d55a15af" PRIMARY KEY ("name"))`, ); } diff --git a/src/shared/models/cron-lease/__tests__/cron-lease.entity.spec.ts b/src/shared/models/cron-lease/__tests__/cron-lease.entity.spec.ts new file mode 100644 index 0000000000..7869070346 --- /dev/null +++ b/src/shared/models/cron-lease/__tests__/cron-lease.entity.spec.ts @@ -0,0 +1,90 @@ +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { DataSource } from 'typeorm'; +import { CronLease } from '../cron-lease.entity'; + +/** + * The table is created by a hand-written migration and read by hand-written SQL, so nothing in the + * running application ever compares the entity against the schema. The next `npm run migration` + * does, and it acts on what it finds: an entity that has drifted from the migration produces a + * migration that "fixes" the difference — in the direction of the entity. + * + * So the entity is checked against the DDL directly, by building the metadata TypeORM would build + * and asking its own driver and naming strategy what column definitions and constraint name that + * yields. No connection is involved; the metadata is derived from the decorators alone. + */ +const MIGRATION = join(__dirname, '..', '..', '..', '..', '..', 'migration', '1785600000000-AddCronLease.js'); + +describe('CronLease entity', () => { + let dataSource: DataSource; + + beforeAll(async () => { + dataSource = new DataSource({ type: 'postgres', entities: [CronLease] }); + + // Builds the entity metadata without connecting to anything. + await (dataSource as unknown as { buildMetadatas: () => Promise }).buildMetadatas(); + }); + + /** The ` [NOT NULL] [DEFAULT ..]` fragment TypeORM would emit for each column. */ + function columnDefinitions(): string[] { + const metadata = dataSource.getMetadata(CronLease); + + return metadata.columns.map((column) => { + const type = dataSource.driver.normalizeType(column); + const length = dataSource.driver.getColumnLength(column); + const fallback = dataSource.driver.normalizeDefault(column); + + return [ + `"${column.databaseName}"`, + length ? `${type}(${length})` : type.toUpperCase(), + column.isNullable ? '' : 'NOT NULL', + fallback ? `DEFAULT ${fallback}` : '', + ] + .filter(Boolean) + .join(' '); + }); + } + + it('maps to the table the migration creates', () => { + expect(dataSource.getMetadata(CronLease).tableName).toEqual('cron_lease'); + }); + + it('declares every column the migration declares, and no other', () => { + const ddl = readFileSync(MIGRATION, 'utf8'); + + for (const definition of columnDefinitions()) { + expect(ddl).toContain(definition); + } + + // The other direction: a column added to the table but not to the entity would be dropped by + // the next generated migration, which the loop above cannot see. + const created = /CREATE TABLE "cron_lease" \((.*?), CONSTRAINT/s.exec(ddl); + + expect(created).not.toBeNull(); + expect(created[1].split(/, (?=")/).length).toEqual(columnDefinitions().length); + }); + + it('gives the primary key the name the migration uses', () => { + const metadata = dataSource.getMetadata(CronLease); + const name = dataSource.namingStrategy.primaryKeyName( + metadata.tableName, + metadata.primaryColumns.map((column) => column.databaseName), + ); + + expect(name).toEqual('PK_a12c181c2b26f33be13d55a15af'); + expect(readFileSync(MIGRATION, 'utf8')).toContain(`CONSTRAINT "${name}" PRIMARY KEY ("name")`); + }); + + it('keeps both timestamps zone-aware', () => { + // They are compared against now() in raw SQL. Without a zone the comparison runs through the + // session time zone, and the lease expires an hour late or an hour early across a daylight + // saving change — the first is a job that runs nowhere, the second is two processes running it. + const metadata = dataSource.getMetadata(CronLease); + + for (const name of ['acquired', 'expires']) { + const column = metadata.columns.find((c) => c.databaseName === name); + + expect(dataSource.driver.normalizeType(column)).toEqual('timestamp with time zone'); + } + }); +}); diff --git a/src/shared/models/cron-lease/cron-lease.entity.ts b/src/shared/models/cron-lease/cron-lease.entity.ts new file mode 100644 index 0000000000..dd7af3d733 --- /dev/null +++ b/src/shared/models/cron-lease/cron-lease.entity.ts @@ -0,0 +1,39 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +/** + * The cross-process claim on a scheduled job. One row per job name; see CronLeaseService, which is + * the only thing that reads or writes it. + * + * It exists as an entity even though the service never goes through a repository. The claim is a + * single `INSERT .. ON CONFLICT .. WHERE`, whose atomicity is the whole point and which the query + * builder cannot express, so the statements stay hand-written. But a table that exists only as DDL + * inside a migration is invisible to the entity model, and the next generated migration would read + * that absence as an instruction: it would carry a `DROP TABLE "cron_lease"`, and the lock would be + * gone without anyone deciding it should be. + * + * The timestamps are `timestamptz`, the only ones in this schema that are. They are compared + * against `now()` in raw SQL rather than mapped through a Date on the way in and out, and a + * `timestamp` on one side of that comparison is resolved through whatever time zone the session + * happens to carry — the same row then expires an hour late or an hour early across a daylight + * saving change, and outright inconsistently between two sessions that disagree. An hour late + * means the job runs nowhere; an hour early means two processes run it at once. + * + * Kept in step with migration/1785600000000-AddCronLease.js by + * src/shared/models/cron-lease/__tests__/cron-lease.entity.spec.ts. + */ +@Entity() +export class CronLease { + /** The job, as `::` — the name DfxCronService registers it under. */ + @PrimaryColumn({ length: 256 }) + name: string; + + /** The process holding it: its role and a per-process random part. */ + @Column({ length: 256 }) + owner: string; + + @Column({ type: 'timestamptz', default: () => 'now()' }) + acquired: Date; + + @Column({ type: 'timestamptz' }) + expires: Date; +} diff --git a/src/shared/shared.module.ts b/src/shared/shared.module.ts index 697f7f5f32..5260a95ad1 100644 --- a/src/shared/shared.module.ts +++ b/src/shared/shared.module.ts @@ -19,6 +19,7 @@ import { CountryController } from './models/country/country.controller'; import { Country } from './models/country/country.entity'; import { CountryRepository } from './models/country/country.repository'; import { CountryService } from './models/country/country.service'; +import { CronLease } from './models/cron-lease/cron-lease.entity'; import { FiatController } from './models/fiat/fiat.controller'; import { Fiat } from './models/fiat/fiat.entity'; import { FiatRepository } from './models/fiat/fiat.repository'; @@ -47,7 +48,10 @@ import { ProcessService } from './services/process.service'; HttpModule, ConfigModule, GeoLocationModule, - TypeOrmModule.forFeature([Asset, Fiat, Country, Language, Setting, IpLog]), + // CronLease has no repository and no service reading it through one — CronLeaseService issues + // the claim as a single statement. It is registered so `autoLoadEntities` knows the table + // belongs to the model; without that a generated migration would offer to drop it. + TypeOrmModule.forFeature([Asset, Fiat, Country, Language, Setting, IpLog, CronLease]), PassportModule.register({ defaultStrategy: 'jwt', session: true }), JwtModule.register(GetConfig().auth.jwt), I18nModule.forRoot(GetConfig().i18n), From 6927066feee461743a54ba2674acbe14c081ce8f Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:03 +0200 Subject: [PATCH 39/86] Put the Spark wallet maintenance behind the same lock as every other job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `SparkClient` drove its token optimization from a `setInterval` of its own, guarded by a role check. A timer outside the scheduler is invisible to the scope, and with it to the cross-process lease — so the role check was the only thing between two processes and the same wallet, and it cannot help in the case that matters: both processes legitimately holding a role that includes this work. That is every deployment, for as long as the outgoing container is up. It is a job on SparkService now, registered through @DfxCron with scope `worker`, which is what routes it through the lease. The client keeps the operation as a plain method, still going through its reconnect-and-retry path. CONTRIBUTING asks for exactly this — periodic work registers through @DfxCron — so the exception the registration guard carried for this file is gone with it, and the guard's own honesty check would fail if it had been left behind. The job declares a `process` flag, per the rule in the inventory, so wallet maintenance can be switched off without a deploy. Inventory recounted from the source rather than adjusted: 138 declarations (137 before), 117 `worker`, 7 `api`, 14 `both`, across 97 files and the same 34 areas. Both distribution tables sum to 138 and the job table has 138 rows. --- docs/cron-jobs.md | 15 +++--- .../spark/__tests__/spark-client.spec.ts | 49 +++++-------------- .../spark/__tests__/spark.service.spec.ts | 28 +++++++++++ .../blockchain/spark/spark-client.ts | 28 +++++------ .../blockchain/spark/spark.service.ts | 26 ++++++++++ src/shared/services/process.service.ts | 1 + 6 files changed, 85 insertions(+), 62 deletions(-) create mode 100644 src/integration/blockchain/spark/__tests__/spark.service.spec.ts diff --git a/docs/cron-jobs.md b/docs/cron-jobs.md index 05fed38bd9..ff8acc2895 100644 --- a/docs/cron-jobs.md +++ b/docs/cron-jobs.md @@ -1,6 +1,6 @@ # Cron jobs -Every scheduled job this service runs: **137 `@DfxCron` declarations** across 96 files and 34 areas. +Every scheduled job this service runs: **138 `@DfxCron` declarations** across 97 files and 34 areas. ## Columns @@ -15,7 +15,7 @@ Every scheduled job this service runs: **137 `@DfxCron` declarations** across 96 ## Scopes `scope` is a mandatory parameter of `@DfxCron` and says which process registers the job: -116 are `worker`, 7 are `api`, 14 are `both`. `CRON_ROLE` decides what a process is +117 are `worker`, 7 are `api`, 14 are `both`. `CRON_ROLE` decides what a process is (`worker`, `api`, or `all` for a single-process setup); a process runs its own scope plus `both`. `worker` is the normal case — anything writing to the database or driving business forward belongs @@ -30,7 +30,7 @@ request path loads on demand, and a job may refresh it but must not be the only ## Flags -116 of the 137 jobs carry a `process` flag, 21 do not. A job with a flag can be switched off +117 of the 138 jobs carry a `process` flag, 21 do not. A job with a flag can be switched off without a deploy — `DfxCronService` skips it when the process appears in the disabled set, which `ProcessService` refreshes from the `disabledProcesses` setting and the `DISABLED_PROCESSES` environment variable every 30 seconds. @@ -64,7 +64,7 @@ New jobs should declare a flag unless there is a reason like the one above. | 10 seconds | 3 | | 30 seconds | 9 | | minute | 52 | -| 5 minutes | 17 | +| 5 minutes | 18 | | 10 minutes | 16 | | hour | 16 | | day at 3am | 1 | @@ -85,7 +85,7 @@ Jobs by area: | `subdomains/core/monitoring` | 14 | — | | `subdomains/core/accounting` | 13 | — | | `subdomains/supporting/payin` | 12 | — | -| `integration/blockchain` | 6 | — | +| `integration/blockchain` | 7 | — | | `subdomains/core/buy-crypto` | 6 | 4 | | `shared/services` | 5 | 5 | | `subdomains/core/sell-crypto` | 5 | 2 | @@ -121,7 +121,7 @@ Jobs by area: Every `@DfxCron(` occurrence in `src/**/*.ts`. Decorator arguments are read by a balanced-paren scan, so multi-line declarations are included — a line-based match misses 26 of them. Interval, flag and scope come from those arguments, so all three are as accurate as the source. The parsed -count is asserted against a raw text count of the decorator: **137 = 137**, no gap. Class and +count is asserted against a raw text count of the decorator: **138 = 138**, no gap. Class and method come from the enclosing `export class` (including `export abstract class`) and the identifier following the decorator. @@ -145,7 +145,7 @@ the job is registered — on the provider instance, which is a different object instance the request handlers use. Resolving either one is a decision about the jobs, not about this inventory, so both are recorded -here rather than fixed in passing. Of the 137 declarations, 136 have a registration path. +here rather than fixed in passing. Of the 138 declarations, 137 have a registration path. ## Jobs @@ -233,6 +233,7 @@ here rather than fixed in passing. Of the 137 declarations, 136 have a registrat | 5 minutes | `LIQUIDITY_MANAGEMENT` | `worker` | `LiquidityManagementRuleService::reactivateRules` | `subdomains/core/liquidity-management/services/liquidity-management-rule.service.ts` | | 5 minutes | `PAY_IN_MAIL` | `worker` | `PayInNotificationService::sendNotificationMails` | `subdomains/supporting/payin/services/payin-notification.service.ts` | | 5 minutes | `REALUNIT_TRANSFER_RECONCILIATION` | `worker` | `RealUnitJobService::reconcilePendingTransfers` | `subdomains/supporting/realunit/realunit-job.service.ts` | +| 5 minutes | `SPARK_TOKEN_OPTIMIZATION` | `worker` | `SparkService::optimizeTokenOutputs` | `integration/blockchain/spark/spark.service.ts` | | 5 minutes | `SUPPORT_BOT` | `worker` | `SupportEscalationService::checkEscalations` | `subdomains/supporting/support-issue/services/support-escalation.service.ts` | | 5 minutes | `TRADING` | `worker` | `TradingJobService::reactivateRules` | `subdomains/core/trading/services/trading-job.service.ts` | | 5 minutes | — | `both` | `TransactionHelper::updateCache` | `subdomains/supporting/payment/services/transaction-helper.ts` | diff --git a/src/integration/blockchain/spark/__tests__/spark-client.spec.ts b/src/integration/blockchain/spark/__tests__/spark-client.spec.ts index 5d6808b2ba..e4d002682b 100644 --- a/src/integration/blockchain/spark/__tests__/spark-client.spec.ts +++ b/src/integration/blockchain/spark/__tests__/spark-client.spec.ts @@ -15,17 +15,9 @@ jest.mock('@buildonspark/spark-sdk', () => ({ SparkWallet: { initialize: jest.fn() }, })); -// The process role, set per test. The name has to start with `mock`, otherwise Jest forbids -// reaching it from the hoisted module factory. -let mockCronRole = 'worker'; - jest.mock('src/config/config', () => ({ - // The module is replaced entirely so the test does not pull in the whole configuration - // chain. CronRole therefore has to come along: the client reads it to bind the optimization - // timer to the role. - CronRole: { ALL: 'all', API: 'api', WORKER: 'worker' }, + // The module is replaced entirely so the test does not pull in the whole configuration chain. GetConfig: () => ({ - cronRole: mockCronRole, blockchain: { spark: { sparkWalletSeed: 'test seed phrase', @@ -72,45 +64,26 @@ describe('SparkClient', () => { afterEach(() => { jest.restoreAllMocks(); - mockCronRole = 'worker'; }); - // --- TOKEN OPTIMIZATION TIMER --- // - - describe('token optimization timer', () => { - it('runs the wallet maintenance in the worker process', () => { - // The timer is not registered through the scheduler, so the client decides itself: under - // the worker role it is created, with the five-minute interval the client sets. - const interval = jest.spyOn(global, 'setInterval'); - - new SparkClient(); - - expect(interval).toHaveBeenCalledTimes(1); - expect(interval.mock.calls[0][1]).toBe(5 * 60 * 1000); - - clearInterval(interval.mock.results[0].value as NodeJS.Timeout); - }); + // --- TOKEN OPTIMIZATION --- // - it('runs it in the single-process role', () => { - // The single-process role must keep the timer: only CronRole.API skips it. - mockCronRole = 'all'; + describe('token optimization', () => { + it('starts no timer of its own', () => { + // Wallet maintenance is a job of SparkService now, registered through @DfxCron. A timer here + // would be invisible to the scheduler and therefore to the scope and the cross-process + // lease, which is how two processes came to optimize the same wallet at once. const interval = jest.spyOn(global, 'setInterval'); new SparkClient(); - expect(interval).toHaveBeenCalledTimes(1); - - clearInterval(interval.mock.results[0].value as NodeJS.Timeout); + expect(interval).not.toHaveBeenCalled(); }); - it('does not run it in the API process', () => { - // The counterpart: under the API role no timer is created at all. - mockCronRole = 'api'; - const interval = jest.spyOn(global, 'setInterval'); + it('optimizes through the reconnecting call path', async () => { + await client.optimizeTokenOutputs(); - new SparkClient(); - - expect(interval).not.toHaveBeenCalled(); + expect(mockWallet.optimizeTokenOutputs).toHaveBeenCalledTimes(1); }); }); diff --git a/src/integration/blockchain/spark/__tests__/spark.service.spec.ts b/src/integration/blockchain/spark/__tests__/spark.service.spec.ts new file mode 100644 index 0000000000..b29113a564 --- /dev/null +++ b/src/integration/blockchain/spark/__tests__/spark.service.spec.ts @@ -0,0 +1,28 @@ +import { DFX_CRONJOB_PARAMS, CronScope, DfxCronParams } from 'src/shared/utils/cron'; +import { SparkService } from '../spark.service'; + +jest.mock('@buildonspark/spark-sdk', () => ({ + SparkWallet: { initialize: jest.fn().mockResolvedValue({ wallet: { on: jest.fn() } }) }, +})); + +/** + * Wallet maintenance moved out of a timer inside SparkClient and into a job here. What that buys + * is not visible at the call site: a job registered through @DfxCron passes the scope filter and + * the cross-process lease, and a timer does neither. These assertions are the only place that + * says so. + */ +describe('SparkService', () => { + const params = (): DfxCronParams => + Reflect.getMetadata(DFX_CRONJOB_PARAMS, SparkService.prototype.optimizeTokenOutputs); + + it('registers the wallet maintenance as a scheduled job', () => { + // Without the decorator it is a plain method nobody calls, and the maintenance stops. + expect(params()).toBeDefined(); + }); + + it('scopes it to the worker, so it goes through the lease', () => { + // `worker` is what puts it behind the cross-process lease — `both` is the one scope exempt + // from it, and would put every process on the same wallet by design. + expect(params().scope).toEqual(CronScope.WORKER); + }); +}); diff --git a/src/integration/blockchain/spark/spark-client.ts b/src/integration/blockchain/spark/spark-client.ts index b20444a454..84d0f4c2b1 100644 --- a/src/integration/blockchain/spark/spark-client.ts +++ b/src/integration/blockchain/spark/spark-client.ts @@ -1,6 +1,6 @@ import { SparkWallet } from '@buildonspark/spark-sdk'; import { Currency } from '@uniswap/sdk-core'; -import { CronRole, GetConfig } from 'src/config/config'; +import { GetConfig } from 'src/config/config'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { AsyncField } from 'src/shared/utils/async-field'; import { BlockchainTokenBalance } from '../shared/dto/blockchain-token-balance.dto'; @@ -52,14 +52,12 @@ export class SparkClient extends BlockchainClient { private wallet: AsyncField; private readonly cachedAddress: AsyncField; private reconnectAttempt = 0; - private tokenOptimizationInterval?: NodeJS.Timeout; constructor() { super(); this.wallet = new AsyncField(() => this.initializeWallet(), true); this.cachedAddress = new AsyncField(() => this.wallet.then((w) => w.getSparkAddress()), true); - this.startTokenOptimization(); } private async call(operation: (wallet: SparkWallet) => Promise): Promise { @@ -225,20 +223,16 @@ export class SparkClient extends BlockchainClient { }); } - private startTokenOptimization(): void { - // On-chain wallet maintenance belongs to exactly one process. This timer is not registered - // through DfxCronService, so it carries the role check itself: under CronRole.API no timer - // is created and optimizeTokenOutputs is never called from here. - if (GetConfig().cronRole === CronRole.API) return; - - if (this.tokenOptimizationInterval) clearInterval(this.tokenOptimizationInterval); - - const intervalMs = 5 * 60 * 1000; // 5 minutes - this.tokenOptimizationInterval = setInterval(() => { - this.call((wallet) => wallet.optimizeTokenOutputs()).catch((e) => { - this.logger.warn('Token optimization failed, will retry on next interval:', e); - }); - }, intervalMs); + /** + * Consolidates the token outputs of the wallet. + * + * Driven by SparkService through @DfxCron rather than by a timer this client starts for itself. + * A timer here is invisible to the scheduler, and with it to the scope and to the cross-process + * lease — two processes would run this against the same wallet whenever their roles overlap, + * which is exactly what a deployment produces while the old container is still up. + */ + async optimizeTokenOutputs(): Promise { + await this.call((wallet) => wallet.optimizeTokenOutputs()); } private reconnectWallet(): void { diff --git a/src/integration/blockchain/spark/spark.service.ts b/src/integration/blockchain/spark/spark.service.ts index aee8bd20e0..bb4d0bea6b 100644 --- a/src/integration/blockchain/spark/spark.service.ts +++ b/src/integration/blockchain/spark/spark.service.ts @@ -1,4 +1,8 @@ import { Injectable } from '@nestjs/common'; +import { CronExpression } from '@nestjs/schedule'; +import { DfxLogger } from 'src/shared/services/dfx-logger'; +import { Process } from 'src/shared/services/process.service'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Bech32mService } from '../shared/bech32m/bech32m.service'; import { SparkClient, SparkTransaction } from './spark-client'; @@ -6,6 +10,8 @@ import { SparkClient, SparkTransaction } from './spark-client'; export class SparkService extends Bech32mService { readonly defaultPrefix = 'spark'; + private readonly logger = new DfxLogger(SparkService); + private readonly client: SparkClient; constructor() { @@ -17,6 +23,26 @@ export class SparkService extends Bech32mService { return this.client; } + /** + * Wallet maintenance: consolidates the token outputs of the Spark wallet. + * + * The client used to run this from a `setInterval` of its own with a role check in front of it. + * A timer outside the scheduler is invisible to the scope AND to the cross-process lease, so the + * role check was the only thing standing between two processes and the same wallet — and it + * cannot help in the case that matters, where both processes legitimately hold a role that + * includes this work. That is every deployment, for as long as the outgoing container is still + * up. Registered here, it goes through the lease like any other worker job. + * + * Errors are logged rather than rethrown: this is best-effort housekeeping, the next run is five + * minutes away, and a wallet that cannot be reached now is not a reason to raise an incident. + */ + @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.WORKER, process: Process.SPARK_TOKEN_OPTIMIZATION }) + async optimizeTokenOutputs(): Promise { + await this.client + .optimizeTokenOutputs() + .catch((e) => this.logger.warn('Token optimization failed, will retry on the next run:', e)); + } + async isHealthy(): Promise { return this.client.isHealthy(); } diff --git a/src/shared/services/process.service.ts b/src/shared/services/process.service.ts index da65245452..d00d31deaa 100644 --- a/src/shared/services/process.service.ts +++ b/src/shared/services/process.service.ts @@ -122,6 +122,7 @@ export enum Process { REF_CLEANUP = 'RefCleanup', JWT_REVOCATION_SYNC = 'JwtRevocationSync', LATEST_BALANCE_CACHE = 'LatestBalanceCache', + SPARK_TOKEN_OPTIMIZATION = 'SparkTokenOptimization', } const safetyProcesses: Process[] = [ From d7b02bdc8ea4bd464589de2b99979b2249506d8e Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:04 +0200 Subject: [PATCH 40/86] Apply the job's own conditions to the statistic it fills at boot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `StatisticService.onModuleInit` called `doUpdate` directly, which is a path around the scheduler — and therefore around every condition the scheduler applies to that job. The scope was the one that mattered: `doUpdate` is scoped `api` because a request path is the only reader of the field it writes, so the worker ran the aggregation queries once per boot for a value no request in that process can read. Being outside the scheduler it was outside the cross-process lease too, so both processes did it and nothing reported it. `DisabledProcess` was skipped the same way, leaving a job switched off through DISABLED_PROCESSES with one run per deployment. And a `void` with no `.catch()` turns a failing query at boot into an unhandled rejection, which main.ts exits the process on. All three now sit in front of the call, the way DashboardFinancialService does it for the same kind of start-up fill. --- .../__tests__/statistic.service.spec.ts | 83 +++++++++++++++++++ .../core/statistic/statistic.service.ts | 28 ++++++- 2 files changed, 108 insertions(+), 3 deletions(-) diff --git a/src/subdomains/core/statistic/__tests__/statistic.service.spec.ts b/src/subdomains/core/statistic/__tests__/statistic.service.spec.ts index 02efff7a79..ce40890fca 100644 --- a/src/subdomains/core/statistic/__tests__/statistic.service.spec.ts +++ b/src/subdomains/core/statistic/__tests__/statistic.service.spec.ts @@ -1,10 +1,12 @@ import { createMock } from '@golevelup/ts-jest'; import { Test, TestingModule } from '@nestjs/testing'; +import { ConfigService, GetConfig } from 'src/config/config'; import { Setting } from 'src/shared/models/setting/setting.entity'; import { SettingService } from 'src/shared/models/setting/setting.service'; import { BuyService } from 'src/subdomains/core/buy-crypto/routes/buy/buy.service'; import { SellService } from 'src/subdomains/core/sell-crypto/route/sell.service'; import { StatisticService } from 'src/subdomains/core/statistic/statistic.service'; +import * as ProcessService from 'src/shared/services/process.service'; import { UserService } from 'src/subdomains/generic/user/models/user/user.service'; describe('StatisticService', () => { @@ -52,4 +54,85 @@ describe('StatisticService', () => { await expect(service.getStatus()).resolves.toEqual({}); }); }); + + /** + * The start-up fill runs outside the scheduler, so none of the conditions the scheduler applies + * to `doUpdate` reached it: not the scope, not the process flag, not any error handling. Each + * test below is one of those. + */ + describe('start-up fill', () => { + const originalRole = process.env.CRON_ROLE; + + function withRole(role: string): void { + process.env.CRON_ROLE = role; + new ConfigService(GetConfig()); + } + + beforeEach(() => { + jest.spyOn(ProcessService, 'DisabledProcess').mockReturnValue(false); + }); + + afterEach(() => { + jest.restoreAllMocks(); + + if (originalRole == null) delete process.env.CRON_ROLE; + else process.env.CRON_ROLE = originalRole; + + new ConfigService(GetConfig()); + }); + + it('does not run in the worker process', () => { + // The job is scoped `api`: a request path is the only reader of the field it writes. Run + // here regardless, the worker spent the aggregation queries once per boot on a value no + // request in that process can read — and outside the lease, so nothing reported it. + withRole('worker'); + const update = jest.spyOn(service, 'doUpdate').mockResolvedValue(undefined); + + service.onModuleInit(); + + expect(update).not.toHaveBeenCalled(); + }); + + it.each(['api', 'all'])('runs in the %s process', (role) => { + // The counterpart: where the job belongs, getAll would answer with undefined until the + // first scheduled run an hour later. + withRole(role); + const update = jest.spyOn(service, 'doUpdate').mockResolvedValue(undefined); + + service.onModuleInit(); + + expect(update).toHaveBeenCalledTimes(1); + }); + + it('stays off when the process flag is off', () => { + // Switching a job off has to switch it off, not leave one run per deployment behind. + jest.spyOn(ProcessService, 'DisabledProcess').mockReturnValue(true); + withRole('api'); + const update = jest.spyOn(service, 'doUpdate').mockResolvedValue(undefined); + + service.onModuleInit(); + + expect(update).not.toHaveBeenCalled(); + }); + + it('reports a failed fill instead of leaving an unhandled rejection', async () => { + // `void this.doUpdate()` without a catch turns a failing query at boot into an unhandled + // rejection. + withRole('api'); + const failure = new Error('aggregation failed'); + jest.spyOn(service, 'doUpdate').mockRejectedValue(failure); + const error = jest.spyOn(service['logger'], 'error').mockImplementation(); + + const unhandled = jest.fn(); + process.on('unhandledRejection', unhandled); + + service.onModuleInit(); + await new Promise((resolve) => setImmediate(resolve)); + + process.off('unhandledRejection', unhandled); + + expect(unhandled).not.toHaveBeenCalled(); + expect(error).toHaveBeenCalledWith('Failed to fill the statistic at start-up:', failure); + }); + }); }); diff --git a/src/subdomains/core/statistic/statistic.service.ts b/src/subdomains/core/statistic/statistic.service.ts index d025b9014f..aa86dbf0a8 100644 --- a/src/subdomains/core/statistic/statistic.service.ts +++ b/src/subdomains/core/statistic/statistic.service.ts @@ -1,8 +1,9 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; -import { Config } from 'src/config/config'; +import { Config, CronRole } from 'src/config/config'; import { SettingService } from 'src/shared/models/setting/setting.service'; -import { Process } from 'src/shared/services/process.service'; +import { DfxLogger } from 'src/shared/services/dfx-logger'; +import { DisabledProcess, Process } from 'src/shared/services/process.service'; import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { SellService } from 'src/subdomains/core/sell-crypto/route/sell.service'; @@ -12,6 +13,8 @@ import { SettingStatus, StatisticDto } from './dto/statistic.dto'; @Injectable() export class StatisticService implements OnModuleInit { + private readonly logger = new DfxLogger(StatisticService); + private statistic: StatisticDto; constructor( @@ -22,7 +25,26 @@ export class StatisticService implements OnModuleInit { ) {} onModuleInit() { - void this.doUpdate(); + // Fills the statistic once at start-up instead of leaving getAll answering with undefined + // until the first scheduled run an hour later. The three conditions below are the ones the + // scheduler applies to the job itself, and this call bypasses the scheduler entirely. + // + // The role first: the job is scoped `api` because a request path is the only reader of the + // field it writes. Run here unconditionally, both processes execute it once at boot — outside + // the cross-process lease, so nothing notices — and the worker spends the aggregation queries + // on a value no request in that process can read. + if (Config.cronRole === CronRole.WORKER) return; + + // Then the flag: a job switched off through DISABLED_PROCESSES has to stay off, including at + // start-up. Otherwise switching it off still leaves one run per deployment. + if (DisabledProcess(Process.UPDATE_STATISTIC)) return; + + void this.doUpdate().catch((e) => + // Not rethrown: an unhandled rejection here takes the process down over a statistic, and the + // scheduled run retries within the hour. Logged rather than swallowed, so the empty response + // in the meantime has a reason on record. + this.logger.error('Failed to fill the statistic at start-up:', e), + ); } @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.API, process: Process.UPDATE_STATISTIC, timeout: 7200 }) From dc5422dccbdbe9404d2a36099c00bda029665388 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:05 +0200 Subject: [PATCH 41/86] Say which way the api scope and the lease were resolved, and report the cost MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The comment on PaymentCronService argued the api scope from the delivery being process-local, and dismissed `Both` because "the lock DfxCronService creates lives in the process". That second half stopped being true when the lease arrived in this branch, and with it the reasoning: the job now goes through a lock that spans processes, so the two halves of the comment contradict each other. One says the job must run wherever a caller waits, the other lets it run in one process only. Both cannot hold. The resolution taken is the lease, because its failure mode is the recoverable one: a caller that was not released is a request that times out, while a duplicated payout or merchant webhook stays duplicated. What that costs is that with more than one API process, callers on the process that lost the lease wait for nothing. So the loss is no longer silent. An api-scoped job losing the race is reported at error level, unlike a worker job, which loses it every cycle by design — the distinction comes from the scope, so no job declares it separately. The comment now states the trade instead of the premise it rested on, and names what an actual fix would be: driving the local delivery from persisted state, which is a change to PaymentLinkPaymentService rather than to a scope. --- .../__tests__/cron-lease.service.spec.ts | 30 ++++++++++++++ .../__tests__/cron-registration.guard.spec.ts | 6 +-- .../__tests__/dfx-cron.service.spec.ts | 39 +++++++++++++++++++ src/shared/services/cron-lease.service.ts | 18 ++++++++- src/shared/services/dfx-cron.service.ts | 11 +++++- .../services/payment-cron.service.ts | 15 ++++++- 6 files changed, 109 insertions(+), 10 deletions(-) diff --git a/src/shared/services/__tests__/cron-lease.service.spec.ts b/src/shared/services/__tests__/cron-lease.service.spec.ts index 422a28addc..efd2f21e6e 100644 --- a/src/shared/services/__tests__/cron-lease.service.spec.ts +++ b/src/shared/services/__tests__/cron-lease.service.spec.ts @@ -296,4 +296,34 @@ describe('CronLeaseService', () => { expect(service.takeFailures()).toEqual({ healthy: true, count: 0, last: undefined }); }); }); + + describe('a lost race that is not routine', () => { + // For a worker job, losing the lease is the mechanism working: the other worker is doing the + // work and the result lands in the database. For a job whose effect exists only inside the + // process that runs it, the same outcome means the effect did not happen where it was needed. + // Nothing distinguishes the two at the lease, so the caller says which it is. + + it('reports the loss for a job whose effect is local to its process', async () => { + const { service } = buildService({ acquire: [] }); + const error = jest.spyOn(service['logger'], 'error').mockImplementation(); + const task = jest.fn(); + + await service.run('PaymentCronService::processExpiredPayments', task, true); + + expect(task).not.toHaveBeenCalled(); + expect(error).toHaveBeenCalledTimes(1); + expect(error.mock.calls[0][0]).toContain('PaymentCronService::processExpiredPayments'); + }); + + it('stays quiet for a job whose result lands in the database', async () => { + // These lose the race every cycle by design — one worker holds the lease, the other does + // not. Reporting that would bury the case above under noise. + const { service } = buildService({ acquire: [] }); + const error = jest.spyOn(service['logger'], 'error').mockImplementation(); + + await service.run('SomeWorkerService::job', jest.fn()); + + expect(error).not.toHaveBeenCalled(); + }); + }); }); diff --git a/src/shared/services/__tests__/cron-registration.guard.spec.ts b/src/shared/services/__tests__/cron-registration.guard.spec.ts index 4dc61bcce5..f7ef002ca3 100644 --- a/src/shared/services/__tests__/cron-registration.guard.spec.ts +++ b/src/shared/services/__tests__/cron-registration.guard.spec.ts @@ -43,12 +43,8 @@ const FORBIDDEN: { pattern: RegExp; what: string; instead: string }[] = [ }, ]; -/** - * A timer tied to the lifetime of an object rather than to a schedule. It is bound to the role - * directly, which is the same decision the scope expresses for a cron job. - */ +/** Timers tied to the lifetime of something other than a schedule. */ const ALLOWED = [ - 'integration/blockchain/spark/spark-client.ts', // The lease renewal is bound to the lifetime of a single job run, not to a schedule: it starts // when that run takes the lease and is cleared in its `finally`. Routing it through @DfxCron // would be circular — it is the mechanism that keeps @DfxCron jobs from running twice. diff --git a/src/shared/services/__tests__/dfx-cron.service.spec.ts b/src/shared/services/__tests__/dfx-cron.service.spec.ts index e1cd751254..6bb3688c3e 100644 --- a/src/shared/services/__tests__/dfx-cron.service.spec.ts +++ b/src/shared/services/__tests__/dfx-cron.service.spec.ts @@ -267,6 +267,45 @@ describe('DfxCronService', () => { expect(claim).toBeDefined(); expect(claim[1][2]).toEqual('60'); }); + + it('marks only the api-scoped jobs as ones whose lost race is worth reporting', async () => { + // The flag is derived from the scope, not declared per job: an api-scoped job is one whose + // effect is confined to the process running it, so losing the lease means that effect did + // not happen. A worker job loses it every cycle by design. + process.env.CRON_ROLE = 'all'; + new ConfigService(GetConfig()); + + const reported = new Map(); + const jobs = [ + providerWithJob('apiJob', { expression: CronExpression.EVERY_MINUTE, scope: CronScope.API, useDelay: false }), + providerWithJob('workerJob', { + expression: CronExpression.EVERY_MINUTE, + scope: CronScope.WORKER, + useDelay: false, + }), + ]; + const discovery = createMock({ + getProviders: () => + jobs.map((p) => ({ ...p, isDependencyTreeStatic: () => true })) as ReturnType< + DiscoveryService['getProviders'] + >, + }); + const metadataScanner = createMock({ getAllMethodNames: (i: object) => Object.keys(i) }); + const leaseSpy = createMock({ + run: (job: string, task: () => Promise, reportContention?: boolean) => { + reported.set(job, reportContention); + return task(); + }, + }); + + new DfxCronService(discovery, metadataScanner, createMock(), leaseSpy).onModuleInit(); + + const scheduled = (CronJob as unknown as jest.Mock).mock.calls.map(([, fn]) => fn as () => unknown); + for (const fire of scheduled) await fire(); + + expect(reported.get('Object::apiJob')).toBe(true); + expect(reported.get('Object::workerJob')).toBe(false); + }); }); describe('role heartbeat', () => { diff --git a/src/shared/services/cron-lease.service.ts b/src/shared/services/cron-lease.service.ts index a2eee003ed..5c9eb0d420 100644 --- a/src/shared/services/cron-lease.service.ts +++ b/src/shared/services/cron-lease.service.ts @@ -180,8 +180,14 @@ export class CronLeaseService implements OnModuleInit, BeforeApplicationShutdown * Failing to reach the database means NOT running: a job that moves money must not proceed on * the assumption that it is probably alone. The caller sees the same outcome as a job whose * lease is held elsewhere — it simply does not run this cycle and tries again on the next. + * + * `reportContention` marks jobs for which losing the race is not a normal outcome. For a worker + * job it is: the other worker holds the lease and is doing the work, and the result lands in the + * database where everyone can see it. For a job whose effect is confined to the process that + * runs it, losing the race means that effect did not happen where it was needed — see + * PaymentCronService. Nothing else can tell the two apart, so the caller says which it is. */ - async run(job: string, task: () => Promise): Promise { + async run(job: string, task: () => Promise, reportContention = false): Promise { let acquired: boolean; try { acquired = await this.acquire(job); @@ -191,7 +197,15 @@ export class CronLeaseService implements OnModuleInit, BeforeApplicationShutdown return; } - if (!acquired) return; + if (!acquired) { + if (reportContention) + this.logger.error( + `Skipped ${job}: another process holds the lease. This job only has an effect in the ` + + `process that runs it, so that effect did not happen here`, + ); + + return; + } // Unref'd: a pending timer must never hold the process open on shutdown. const renewal = setInterval(() => { diff --git a/src/shared/services/dfx-cron.service.ts b/src/shared/services/dfx-cron.service.ts index 13ac2a35d6..45cdb8e66c 100644 --- a/src/shared/services/dfx-cron.service.ts +++ b/src/shared/services/dfx-cron.service.ts @@ -161,13 +161,22 @@ export class DfxCronService implements OnModuleInit { * starve whichever process lost the race, and the job would silently stop maintaining that * state. Their safety comes from a different property: running twice must be harmless by * construction, which is what CONTRIBUTING requires of them. + * + * `API` jobs are leased, and reported when they lose the race. They are declared `API` because + * their effect is confined to the process running them, which is an argument for running them + * in every API process; they also write to the database and call out, which is an argument for + * running them once. Only one of those can be had. The lease is the side that cannot be undone + * afterwards — a duplicated payout or webhook stays duplicated, whereas a caller that was not + * released is a request that times out. Losing the race therefore stays possible by design, and + * is reported rather than passed over, because for these jobs it is the symptom of a deployment + * running more than one API process rather than a normal cycle. */ private guardAcrossProcesses(cronJobName: string, data: CronJobData): () => Promise { const task = this.wrapFunction(data); if (data.params.scope === CronScope.BOTH) return task; - return () => this.leases.run(cronJobName, task); + return () => this.leases.run(cronJobName, task, data.params.scope === CronScope.API); } private wrapFunction(data: CronJobData) { diff --git a/src/subdomains/core/payment-link/services/payment-cron.service.ts b/src/subdomains/core/payment-link/services/payment-cron.service.ts index 69337009db..31e4b10ba4 100644 --- a/src/subdomains/core/payment-link/services/payment-cron.service.ts +++ b/src/subdomains/core/payment-link/services/payment-cron.service.ts @@ -21,8 +21,19 @@ export class PaymentCronService { // waitForPayment awaits and pushes the device activation into the RxJS subject PaymentLinkGateway // subscribes to. Both hold their state in the instance, so they only reach a caller of this same // process. `Both` is no option either: the jobs write to the database and trigger merchant - // webhooks, which a second registration would repeat - the lock DfxCronService creates lives in - // the process, so it cannot prevent a second process from running the same job. + // webhooks, which a second registration would repeat. + // + // Those two properties pull against each other, and the scope alone cannot settle it. "Only + // reaches a caller of this process" argues for running in every API process; "writes and calls + // out" argues for running in exactly one. DfxCronService leases these jobs, so it is the second: + // with more than one API process, the one that loses the lease does not release the callers + // waiting on it, and they wait until their own timeout. That is the recoverable side — a + // duplicated payout or merchant webhook is not. The lease reports the lost race at error level + // for exactly this reason; see DfxCronService.guardAcrossProcesses. + // + // Resolving it properly means driving the local delivery from persisted state rather than from + // the job that does the writing, so any process can release its own waiters. That is a change to + // PaymentLinkPaymentService, not to this scope. @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.API, process: Process.PAYMENT_EXPIRATION }) async processExpiredPayments(): Promise { await this.paymentLinkPaymentService.processExpiredPayments(); From e2c0c47df6d74a61ff4022afda0a8448993287db Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:07 +0200 Subject: [PATCH 42/86] Do not make the monitoring endpoints depend on a row with id 1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The read path moved from "the highest id" to `where: { id: 1 }`, which is the row the write path targets — correct, because reading by highest id lets a second row make every read miss what is written. What it does not survive is an environment whose state does not happen to live under id 1: `readState` returns null forever, and since the observers are scoped to the worker, the API process never fills the state itself. The monitoring endpoints would answer 404 indefinitely. Which environments have that row is a fact about their databases, not about this repository, so the code no longer assumes either way. Both paths now prefer id 1 and fall back to the highest id. The write path uses the same lookup deliberately: seeding a fresh id 1 with only the metrics this process changed would strand everything the old row held, and the reads, which prefer id 1, would then answer from the partial row. Seeding from what it finds converges instead — after the first write the row exists and the fallback stops being reached. --- .../__tests__/monitoring.service.spec.ts | 63 +++++++++++++++++++ .../core/monitoring/monitoring.service.ts | 45 ++++++++++--- 2 files changed, 100 insertions(+), 8 deletions(-) diff --git a/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts b/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts index 4b48e5e288..68245cd6cc 100644 --- a/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts +++ b/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts @@ -228,4 +228,67 @@ describe('MonitoringService', () => { expect(result.bank.balance.data).toEqual({ chf: 42 }); }); }); + + describe('an environment whose state row is not id 1', () => { + // The write path targets id 1, so the read has to prefer it — reading the highest id would let + // a second row make every read miss what is written. But whether a given database HAS that row + // is not decidable from here, and reading only id 1 would answer null forever where it does + // not: the observers are scoped to the worker, so the API process would never fill the state + // itself and the monitoring endpoints would answer 404 indefinitely. + + /** A repository holding one row under a different id. */ + function withRowAt(id: number): void { + repo.findOne.mockImplementation((options: { where?: { id?: number } }) => + Promise.resolve(options?.where?.id === 1 ? null : ({ id, data: JSON.stringify(persisted) } as never)), + ); + } + + it('answers from the row that exists instead of reporting nothing', async () => { + withRowAt(7); + + await expect(service.getState(undefined, undefined)).resolves.toEqual(asRead()); + }); + + it('still reports nothing when there is no row at all', async () => { + repo.findOne.mockResolvedValue(null as never); + + await expect(service.getState('node', 'health')).rejects.toBeInstanceOf(NotFoundException); + }); + + it('seeds id 1 with what the old row held, not only with what this process changed', async () => { + // Writing just the changed metric into a fresh id 1 would strand everything the old row + // carried — and the reads, which prefer id 1, would then answer from the partial row. + const managerFindOne = jest + .fn() + .mockImplementation((_entity: unknown, options: { where?: { id?: number } }) => + Promise.resolve(options?.where?.id === 1 ? null : { id: 7, data: JSON.stringify(persisted) }), + ); + Object.defineProperty(repo, 'manager', { + value: { + transaction: (run: (m: unknown) => Promise) => + run({ + findOne: managerFindOne, + save: jest.fn().mockImplementation((_e: unknown, row: { id: number; data: string }) => { + written.push(row); + return Promise.resolve(row); + }), + }), + }, + configurable: true, + }); + + await service['mergeIntoStoredState']([['aml', 'freeze']], { + aml: { freeze: metric({ frozen: 0 }, '2030-01-01T00:00:00Z') }, + }); + + expect(written).toHaveLength(1); + expect(written[0].id).toEqual(1); + + const saved = JSON.parse(written[0].data); + + expect(saved.aml.freeze.data).toEqual({ frozen: 0 }); + expect(saved.node.health.data).toEqual({ up: true }); + expect(saved.bank.balance.data).toEqual({ chf: 42 }); + }); + }); }); diff --git a/src/subdomains/core/monitoring/monitoring.service.ts b/src/subdomains/core/monitoring/monitoring.service.ts index eeda07b857..6d59d21f94 100644 --- a/src/subdomains/core/monitoring/monitoring.service.ts +++ b/src/subdomains/core/monitoring/monitoring.service.ts @@ -1,5 +1,6 @@ import { Injectable, NotFoundException, OnModuleInit } from '@nestjs/common'; import { cloneDeep, isEqual } from 'lodash'; +import { FindOneOptions } from 'typeorm'; import { BehaviorSubject, debounceTime, pairwise } from 'rxjs'; import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Util } from 'src/shared/utils/util'; @@ -184,9 +185,12 @@ export class MonitoringService implements OnModuleInit { private async mergeIntoStoredState(changed: [string, string][], newState: SystemState): Promise { return this.systemStateSnapshotRepo.manager.transaction(async (manager) => { - const row = await manager.findOne(SystemStateSnapshot, { - where: { id: 1 }, - lock: { mode: 'pessimistic_write' }, + // Same lookup as the read path, so a row that lives under a different id is merged into + // rather than left behind: writing only what this process changed into a fresh `id: 1` + // would strand every metric the old row carried, and the reads would then prefer the new + // partial row over the complete one. + const row = await this.stateRow((options) => manager.findOne(SystemStateSnapshot, options), { + mode: 'pessimistic_write', }); const stored: SystemState = row ? JSON.parse(row.data) : {}; @@ -230,16 +234,41 @@ export class MonitoringService implements OnModuleInit { /** Reads the persisted state without notifying: unlike loadState, this runs on every read. */ private async readState(): Promise { - // Reads the row the write path targets. The previous `order: { id: 'DESC' }` took the highest - // id instead, so a second row would have made this read one that is never written. - const latestPersistedState = await this.systemStateSnapshotRepo.findOne({ where: { id: 1 } }); + const row = await this.stateRow((options) => this.systemStateSnapshotRepo.findOne(options)); - if (!latestPersistedState) { + if (!row) { this.logger.warn('No monitoring state found in the database'); return null; } - return JSON.parse(latestPersistedState.data); + return JSON.parse(row.data); + } + + /** + * The row holding the system state: `id: 1` where it exists, otherwise the highest id there is. + * + * The write path targets `id: 1`, and reading by highest id instead would let a second row make + * every read miss what is written. But whether an environment has that row is a property of its + * database, not of this code — and reading only `id: 1` would answer null forever wherever it + * does not, which since the observers are scoped to the worker means the monitoring endpoints + * would answer 404 on the API process indefinitely. + * + * So the read falls back, and the write path uses the same lookup to seed `id: 1` from what it + * finds. That converges: after the first write the row exists and the fallback stops being + * reached. Taking the highest id rather than the lowest keeps the fallback on the row the code + * before this branch wrote to. + */ + private async stateRow( + find: (options: FindOneOptions) => Promise, + lock?: FindOneOptions['lock'], + ): Promise { + const canonical = await find({ where: { id: 1 }, lock }); + if (canonical) return canonical; + + const fallback = await find({ where: {}, order: { id: 'DESC' }, lock }); + if (fallback) this.logger.warn(`No monitoring state under id 1, using id ${fallback.id} instead`); + + return fallback ?? undefined; } /** From 35a8a1d12ef26c9ca3aed27cfdc19c17f1e2339e Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:08 +0200 Subject: [PATCH 43/86] Pin the heartbeat wording the lease alert reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The alert matches the state appended directly to the heartbeat line, so what it needs is the adjacency, not the two substrings. Reported separately — a state on a line of its own, or elsewhere in the same line — the alert stays silent while looser assertions keep passing. One expression pins it; putting the state on its own line makes this test fail. --- src/shared/services/__tests__/dfx-cron.service.spec.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/shared/services/__tests__/dfx-cron.service.spec.ts b/src/shared/services/__tests__/dfx-cron.service.spec.ts index 6bb3688c3e..ac786ed0d3 100644 --- a/src/shared/services/__tests__/dfx-cron.service.spec.ts +++ b/src/shared/services/__tests__/dfx-cron.service.spec.ts @@ -377,7 +377,12 @@ describe('DfxCronService', () => { // Still the shape the role alert matches — a heartbeat that stops matching would read as a // dead process and hide the reason rather than name it. expect(line).toMatch(/CronRole (api|worker|all): heartbeat, [0-9]+ jobs registered/); - expect(line).toContain('lease unusable'); + + // And the shape the lease alert matches, which is the same prefix with the state appended + // directly to it. Pinned as one expression rather than two `toContain`s: what the alert + // needs is the ADJACENCY — a state reported somewhere else in the line, or in a line of its + // own, would leave that alert silent while every looser assertion still passed. + expect(line).toMatch(/CronRole (api|worker|all): heartbeat, [0-9]+ jobs registered, lease unusable/); expect(line).toContain('relation "cron_lease" does not exist'); }); From cfe1d870f9271182a9a9f42c646d76b4d313d0e8 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:09 +0200 Subject: [PATCH 44/86] Wire the lease shutdown to the signal instead of to Nest's global hooks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first version of this reached for `app.enableShutdownHooks()`, which is the idiomatic call and the wrong one here. That switch is global, and this application carries nine `onModuleDestroy` implementations that have never run, because nothing ever asked for a shutdown hook. Nest runs those BEFORE the lease hook, and they empty the strategy registries that PayIn, PayOut and DEX jobs resolve from. On its own that would be harmless — the process was about to die anyway. Next to this change it is not: the whole point of the shutdown path is to keep in-flight jobs alive LONGER into the shutdown, up to ten seconds, so a running payout would now have time to fail on an emptied registry instead of simply being cut off. That is a different, less predictable ending than the one it had before, and handing a lease over sooner is not worth it. So the release hangs off a SIGTERM/SIGINT handler that touches nothing else. Registering one means Node no longer terminates on the signal by itself, so the handler exits: the wait is bounded by its own grace period, and a second signal takes the impatient path rather than holding the container until SIGKILL. Two tests pin it — that bootstrap wires it at all, and that it is NOT wired through `enableShutdownHooks`, since that is exactly what the next reader would reach for. The second one drops comment lines first, or it would fail on the paragraph explaining itself. --- src/main.ts | 51 ++++++++++++++++--- .../__tests__/cron-lease.service.spec.ts | 32 +++++++++--- src/shared/services/cron-lease.service.ts | 21 +++++--- 3 files changed, 84 insertions(+), 20 deletions(-) diff --git a/src/main.ts b/src/main.ts index da229f83e4..2970b6f41f 100644 --- a/src/main.ts +++ b/src/main.ts @@ -5,7 +5,7 @@ import './tracing'; import './runtime-metrics'; // event loop saturation gauges; must follow ./tracing (needs its meter provider) import './polyfills'; // registers global EventSource for @arkade-os/sdk; see src/polyfills.ts -import { VersioningType } from '@nestjs/common'; +import { INestApplication, VersioningType } from '@nestjs/common'; import { NestFactory } from '@nestjs/core'; import { WsAdapter } from '@nestjs/platform-ws'; import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; @@ -22,6 +22,7 @@ import { Config, Environment } from './config/config'; import { ApiExceptionFilter } from './shared/filters/exception.filter'; import { apiTraceMiddleware, maskUrl } from './shared/middlewares/api-trace.middleware'; import { DetailedValidationPipe } from './shared/pipes/detailed-validation.pipe'; +import { CronLeaseService } from './shared/services/cron-lease.service'; import { DfxLogger } from './shared/services/dfx-logger'; import { AccountChangedWebhookDto } from './subdomains/generic/user/services/webhook/dto/account-changed-webhook.dto'; import { @@ -81,11 +82,7 @@ async function bootstrap() { app.use(apiTraceMiddleware()); } - // Without this, Nest never runs a shutdown hook and a deployment kills the process mid-job. The - // cron lease depends on it: see CronLeaseService.beforeApplicationShutdown. Restricted to the - // two signals a deployment actually sends, rather than the full default set — the others are - // crash signals whose default handling should stay untouched. - app.enableShutdownHooks(['SIGTERM', 'SIGINT']); + releaseCronLeasesOnShutdown(app); app.useWebSocketAdapter(new WsAdapter(app)); @@ -136,6 +133,48 @@ async function bootstrap() { new DfxLogger('Main').info(`Application ready ...`); } +/** + * Gives the cron leases a chance to be handed over before a deployment takes the process away. + * + * Nothing in this application ever asked for a shutdown hook, so SIGTERM used to end the process + * instantly: a job running at that moment never reached the release in its `finally`, and its row + * in `cron_lease` sat there until it expired. That is what CronLeaseService.shutdown addresses. + * + * Deliberately a signal handler rather than `app.enableShutdownHooks()`. That switch is global and + * would, for the first time, start running the nine `onModuleDestroy` hooks this application + * carries — Nest runs them BEFORE the hook above, and they empty the strategy registries that + * PayIn, PayOut and DEX jobs resolve from. Since the whole point of the wait is to keep in-flight + * jobs alive longer into the shutdown, the two together would let a running payout fail on an + * emptied registry rather than simply be cut off. Handing a lease over is not worth that. + * + * Registering a handler means Node no longer terminates on the signal by itself, so this has to + * exit. `CronLeaseService.shutdown` is bounded by its own grace period, and a second signal takes + * the impatient path — otherwise a stuck shutdown would hold the container until SIGKILL. + */ +function releaseCronLeasesOnShutdown(app: INestApplication): void { + const logger = new DfxLogger('Shutdown'); + const leases = app.get(CronLeaseService); + + let started = false; + + for (const signal of ['SIGTERM', 'SIGINT'] as const) { + process.on(signal, () => { + if (started) { + logger.warn(`Second ${signal}, exiting without waiting for the running jobs`); + process.exit(1); + } + + started = true; + logger.info(`${signal} received, releasing the cron leases`); + + void leases + .shutdown() + .catch((e) => logger.error('Failed to release the cron leases on shutdown:', e)) + .finally(() => process.exit(0)); + }); + } +} + function runSeed(): void { const logger = new DfxLogger('Seed'); const seedPath = join(process.cwd(), 'migration', 'seed', 'seed.js'); diff --git a/src/shared/services/__tests__/cron-lease.service.spec.ts b/src/shared/services/__tests__/cron-lease.service.spec.ts index efd2f21e6e..1dd7d1c6a1 100644 --- a/src/shared/services/__tests__/cron-lease.service.spec.ts +++ b/src/shared/services/__tests__/cron-lease.service.spec.ts @@ -176,7 +176,7 @@ describe('CronLeaseService', () => { await settle(); - const shutdown = service.beforeApplicationShutdown(); + const shutdown = service.shutdown(); await settle(); // Grace expires with the job still working. @@ -204,7 +204,7 @@ describe('CronLeaseService', () => { await settle(); let over = false; - const shutdown = service.beforeApplicationShutdown().then(() => (over = true)); + const shutdown = service.shutdown().then(() => (over = true)); await settle(); expect(over).toBe(false); @@ -218,14 +218,30 @@ describe('CronLeaseService', () => { expect(released(onQuery)).toBe(true); }); - it('is reached at all, because bootstrap enables the hooks', () => { - // Nest calls no shutdown hook unless the application asks for it. Without that one line - // everything above is dead code and a deployment kills the process mid-job, which is the - // state this change came from. Read from the source because there is nothing to call. + it('is reached at all, because bootstrap wires it to the signal', () => { + // Nothing calls this on its own. Without the wiring in bootstrap everything above is dead + // code and a deployment kills the process mid-job, which is the state this change came + // from. Read from the source because there is nothing to call. const main = readFileSync(join(__dirname, '..', '..', '..', 'main.ts'), 'utf8'); - expect(main).toMatch(/app\.enableShutdownHooks\(/); + expect(main).toContain('releaseCronLeasesOnShutdown(app)'); expect(main).toContain("'SIGTERM'"); + expect(main).toMatch(/leases\s*\n?\s*\.shutdown\(\)/); + }); + + it("is NOT wired through Nest's global shutdown hooks", () => { + // `enableShutdownHooks` would also start running nine `onModuleDestroy` hooks that have + // never run here, before this one, emptying the strategy registries that PayIn, PayOut and + // DEX jobs resolve from — while the wait above deliberately keeps those jobs alive longer. + // Pinned because the idiomatic call is exactly what a later reader would reach for. Comment + // lines are dropped first: the reason for not calling it is written down right next to the + // wiring, and a check that cannot tell the two apart would fail on its own explanation. + const main = readFileSync(join(__dirname, '..', '..', '..', 'main.ts'), 'utf8') + .split('\n') + .filter((line) => !/^\s*(\/\/|\*|\/\*)/.test(line)) + .join('\n'); + + expect(main).not.toMatch(/app\.enableShutdownHooks\(/); }); it('returns immediately when no job is running', async () => { @@ -233,7 +249,7 @@ describe('CronLeaseService', () => { // spend the grace period waiting for it. const { service } = buildService({}); - await service.beforeApplicationShutdown(); + await service.shutdown(); }); it('stops tracking a run once it is done, so a later shutdown has nothing to wait for', async () => { diff --git a/src/shared/services/cron-lease.service.ts b/src/shared/services/cron-lease.service.ts index 5c9eb0d420..7deaf02a31 100644 --- a/src/shared/services/cron-lease.service.ts +++ b/src/shared/services/cron-lease.service.ts @@ -1,4 +1,4 @@ -import { BeforeApplicationShutdown, Injectable, OnModuleInit } from '@nestjs/common'; +import { Injectable, OnModuleInit } from '@nestjs/common'; import { randomUUID } from 'crypto'; import { Config } from 'src/config/config'; import { DataSource } from 'typeorm'; @@ -28,7 +28,7 @@ const LEASE_TTL_SECONDS = 60; const RENEWAL_INTERVAL_MS = (LEASE_TTL_SECONDS / 3) * 1000; /** - * How long shutdown waits for jobs that are still running. See `beforeApplicationShutdown`. + * How long shutdown waits for jobs that are still running. See `shutdown`. * * Short on purpose: it is a handover courtesy, not a completion guarantee. Every process pays it * on every deployment, and the container's own stop grace period ends the process regardless. @@ -59,7 +59,7 @@ const SHUTDOWN_GRACE_MS = 10 * 1000; * more. */ @Injectable() -export class CronLeaseService implements OnModuleInit, BeforeApplicationShutdown { +export class CronLeaseService implements OnModuleInit { private readonly logger = new DfxLogger(CronLeaseService); private ownerId?: string; @@ -245,8 +245,14 @@ export class CronLeaseService implements OnModuleInit, BeforeApplicationShutdown * Waits for the jobs this process is still running, so their normal release path can hand the * lease over to the successor instead of leaving it to expire. * - * `beforeApplicationShutdown`, not `onApplicationShutdown`: TypeOrmCoreModule closes the - * connection in the latter, and releasing a lease needs that connection. + * Called from a SIGTERM/SIGINT handler in `main.ts`, deliberately NOT through Nest's + * `enableShutdownHooks`. That switch is global: it would also start running nine + * `onModuleDestroy` hooks that have never run in this application, because nothing ever asked + * for a shutdown hook. Nest runs those BEFORE this one, and they empty the strategy registries + * that PayIn, PayOut and DEX jobs resolve from. Combined with the wait below — which is the + * whole point here, keeping in-flight jobs alive LONGER into the shutdown — that would let a + * running payout fail on an emptied registry instead of simply being cut off. Handing over a + * lease is not worth activating that. * * A lease is NOT taken away from a job that is still working. Releasing on SIGTERM would hand * over faster, but the job keeps running until the container's stop grace period ends it — @@ -257,8 +263,11 @@ export class CronLeaseService implements OnModuleInit, BeforeApplicationShutdown * What is still running after the wait therefore keeps its lease, which lapses within * `LEASE_TTL_SECONDS` of the last renewal. The renewal timers deliberately keep going meanwhile: * they hold the claim for as long as this process is alive to renew it. + * + * Bounded on every path: the only thing awaited is a race against `SHUTDOWN_GRACE_MS`, so a + * database that has stopped answering cannot turn this into a process that never exits. */ - async beforeApplicationShutdown(): Promise { + async shutdown(): Promise { const running = [...this.inFlight.values()]; if (!running.length) return; From 2959b312af5e95e8f763ecb568df00cf4d2a71d9 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:10 +0200 Subject: [PATCH 45/86] Keep the lease comments to what this repository can vouch for A comment that names a production value, a property of the production database or an alert defined in another repository asserts something this repository cannot check. The connection-budget argument holds without the number, the foreign-key decision holds on its own terms, and the heartbeat test says what reads the line without naming it. --- migration/1785600000000-AddCronLease.js | 4 ++-- src/shared/services/__tests__/dfx-cron.service.spec.ts | 2 +- src/shared/services/cron-lease.service.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/migration/1785600000000-AddCronLease.js b/migration/1785600000000-AddCronLease.js index 82b2736a4f..0258ff17e0 100644 --- a/migration/1785600000000-AddCronLease.js +++ b/migration/1785600000000-AddCronLease.js @@ -16,8 +16,8 @@ * This table makes it structural: a job scoped to exactly one process must hold a row here for the * duration of its run, and the row is claimable by only one process at a time. * - * No foreign keys, deliberately. The table is infrastructure, not domain data, and PRD carries - * tables without a primary key from the MSSQL cutover — a FK into one of them would fail at boot. + * No foreign keys, deliberately: the table is infrastructure, not domain data, and a key into a + * domain table would tie a coordination row to a schema it has no business depending on. * `name` is the primary key, so the claim is a single atomic upsert with no index to keep in sync. * * The primary key carries the name TypeORM derives for it, per the rule in CONTRIBUTING.md: `PK_` diff --git a/src/shared/services/__tests__/dfx-cron.service.spec.ts b/src/shared/services/__tests__/dfx-cron.service.spec.ts index ac786ed0d3..036d044831 100644 --- a/src/shared/services/__tests__/dfx-cron.service.spec.ts +++ b/src/shared/services/__tests__/dfx-cron.service.spec.ts @@ -309,7 +309,7 @@ describe('DfxCronService', () => { }); describe('role heartbeat', () => { - // The alert dfx-api-role-mismatch decides from this line which role each process is running. + // A watchdog outside this repository decides from this line which role each process is running. // Everything it needs has to be IN the line and the line has to appear in both processes — // the three tests below pin exactly that, because none of it is visible at the call site. diff --git a/src/shared/services/cron-lease.service.ts b/src/shared/services/cron-lease.service.ts index 7deaf02a31..f76a19a9a4 100644 --- a/src/shared/services/cron-lease.service.ts +++ b/src/shared/services/cron-lease.service.ts @@ -47,8 +47,8 @@ const SHUTDOWN_GRACE_MS = 10 * 1000; * The lease is claimed per job name, and only one owner can hold it. It carries an expiry rather * than a lock held on a connection: a connection-bound `pg_advisory_lock` would occupy one pooled * connection for the whole runtime of the job, and 67 of the jobs declare a timeout measured in - * minutes. Against `SQL_POOL_MAX=40` that is a real risk to the connection budget. An expiring row - * costs one short query to take, one to extend, one to release. + * minutes. That is a real risk to a connection pool sized by `SQL_POOL_MAX`. An expiring row costs + * one short query to take, one to extend, one to release. * * **What it does not do.** If the database becomes unreachable while a job runs, the lease cannot * be extended and eventually expires — a second process may then start the same job while the From 8c8496aedd114395dcab53a395b1285858620d2d Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:12 +0200 Subject: [PATCH 46/86] Deliver payment updates from the stored state instead of from the writing job The AsyncMap behind waitForPayment and the subject behind the device activation are both process-local, so today only the process that writes a payment releases anyone waiting for it. deliverPaymentUpdates reads the state back for the payments this process is waiting on and for the devices connected to it, which lets any process release its own. It writes nothing and calls nothing outside the process, and it is bounded by what this process holds: with no caller waiting and no device connected it issues no query. doSave keeps delivering directly, so a process that writes and waits at the same time is as immediate as before. The lookup by deviceId is new, so the column gets the index it needs. --- ...0000-AddPaymentLinkPaymentDeviceIdIndex.js | 56 +++++ .../controllers/payment-link.gateway.ts | 6 + .../payment-link-payment.entity.spec.ts | 65 ++++++ .../entities/payment-link-payment.entity.ts | 15 ++ .../payment-link-payment.service.spec.ts | 203 ++++++++++++++++++ .../services/payment-link-payment.service.ts | 135 +++++++++++- 6 files changed, 477 insertions(+), 3 deletions(-) create mode 100644 migration/1785620000000-AddPaymentLinkPaymentDeviceIdIndex.js create mode 100644 src/subdomains/core/payment-link/entities/__tests__/payment-link-payment.entity.spec.ts create mode 100644 src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts diff --git a/migration/1785620000000-AddPaymentLinkPaymentDeviceIdIndex.js b/migration/1785620000000-AddPaymentLinkPaymentDeviceIdIndex.js new file mode 100644 index 0000000000..a5d4b65dc6 --- /dev/null +++ b/migration/1785620000000-AddPaymentLinkPaymentDeviceIdIndex.js @@ -0,0 +1,56 @@ +/** + * @typedef {import('typeorm').MigrationInterface} MigrationInterface + * @typedef {import('typeorm').QueryRunner} QueryRunner + */ + +/** + * Index `payment_link_payment ("deviceId")`. + * + * `PaymentLinkPaymentService.deliverToConnectedDevices` introduces the first lookup by that + * column: while this process holds a websocket connection open for a device, it asks every 15 + * seconds whether a payment of that device has reached a state the device has to be told about. + * `deviceId` carried no index, so that lookup would scan the whole table on every one of those + * ticks for as long as a device stays connected. + * + * A single-column index is enough for the shape of the query. It filters `deviceId IN (…)` on the + * handful of devices connected to this process and narrows the result further by `updated` and by + * status — but a device has few payments, so the index gets the row count down to that handful + * before the remaining conditions are applied. + * + * The name is the deterministic one TypeORM's `DefaultNamingStrategy` derives, since CONTRIBUTING + * disallows custom index names: `IDX_` followed by the first 26 hex characters of + * `sha1('payment_link_payment_deviceId')` (table name + `_` + the column name). It is pinned + * against the entity in `payment-link-payment.entity.spec.ts`, so a rename on either side fails a + * test rather than producing a second index the next generated migration would add. + * + * `CREATE INDEX CONCURRENTLY` is not used: migrations run inside a transaction (`migrationsRun` in + * `src/config/config.ts`, TypeORM's default `migrationsTransactionMode: 'all'`), and CONCURRENTLY + * is not allowed there. The plain form takes a SHARE lock, which blocks writes to the table — and + * because locks are released at COMMIT, it holds until the whole pending batch commits. + * + * @class + * @implements {MigrationInterface} + */ +module.exports = class AddPaymentLinkPaymentDeviceIdIndex1785620000000 { + name = 'AddPaymentLinkPaymentDeviceIdIndex1785620000000'; + + /** + * @param {QueryRunner} queryRunner + */ + async up(queryRunner) { + // SET LOCAL is scoped to the whole transaction. Bounds WAIT time to acquire the lock, not how + // long the lock is held. Set once: this migration has a single CREATE INDEX statement. + await queryRunner.query(`SET LOCAL lock_timeout = '5s'`); + await queryRunner.query(`CREATE INDEX "IDX_8a9b97a10b3db9c64d45ae4d38" ON "payment_link_payment" ("deviceId")`); + } + + /** + * @param {QueryRunner} queryRunner + */ + async down(queryRunner) { + // SET LOCAL is scoped to the whole transaction. Bounds WAIT time to acquire the lock, not how + // long the lock is held. Set once: this migration has a single DROP INDEX statement. + await queryRunner.query(`SET LOCAL lock_timeout = '5s'`); + await queryRunner.query(`DROP INDEX "public"."IDX_8a9b97a10b3db9c64d45ae4d38"`); + } +}; diff --git a/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts b/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts index 6ca46cb14e..2dcdd88461 100644 --- a/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts +++ b/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts @@ -32,6 +32,10 @@ export class PaymentLinkGateway implements OnGatewayConnection, OnModuleInit { clients.set(clientId, client); this.clients.set(device, clients); + // Tells the service which devices this process can still deliver to, so its catch-up job knows + // what to look up and can stay away from the database when nothing is connected here. + this.paymentService.registerDevice(device); + client.onclose = () => this.removeClient(device, clientId); } @@ -39,6 +43,8 @@ export class PaymentLinkGateway implements OnGatewayConnection, OnModuleInit { const clients = this.clients.get(device); clients?.delete(clientId); this.clients.set(device, clients); + + this.paymentService.unregisterDevice(device); } private sendMessage(device: PaymentDevice) { diff --git a/src/subdomains/core/payment-link/entities/__tests__/payment-link-payment.entity.spec.ts b/src/subdomains/core/payment-link/entities/__tests__/payment-link-payment.entity.spec.ts new file mode 100644 index 0000000000..3d21ada445 --- /dev/null +++ b/src/subdomains/core/payment-link/entities/__tests__/payment-link-payment.entity.spec.ts @@ -0,0 +1,65 @@ +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { DefaultNamingStrategy, getMetadataArgsStorage } from 'typeorm'; +import { PaymentLinkPaymentMode, PaymentLinkPaymentStatus } from '../../enums'; +import { PaymentLinkPayment } from '../payment-link-payment.entity'; + +const MIGRATION = join( + __dirname, + '..', + '..', + '..', + '..', + '..', + '..', + 'migration', + '1785620000000-AddPaymentLinkPaymentDeviceIdIndex.js', +); + +describe('PaymentLinkPayment', () => { + function payment(values: Partial): PaymentLinkPayment { + return Object.assign(new PaymentLinkPayment(), { + status: PaymentLinkPaymentStatus.PENDING, + mode: PaymentLinkPaymentMode.SINGLE, + txCount: 0, + ...values, + }); + } + + describe('waitState', () => { + it('changes when the payment leaves pending', () => { + expect(payment({ status: PaymentLinkPaymentStatus.COMPLETED }).waitState).not.toEqual(payment({}).waitState); + }); + + it('changes when a MULTIPLE-mode payment counts another completed quote', () => { + const before = payment({ mode: PaymentLinkPaymentMode.MULTIPLE, txCount: 1 }); + const after = payment({ mode: PaymentLinkPaymentMode.MULTIPLE, txCount: 2 }); + + expect(after.waitState).not.toEqual(before.waitState); + }); + + it('stays the same across changes a caller is not waiting for', () => { + expect(payment({ isConfirmed: true, note: 'edited' }).waitState).toEqual(payment({}).waitState); + }); + }); + + describe('deviceId index', () => { + /** + * `deliverToConnectedDevices` looks payments up by `deviceId`, and the index backing that is + * created by a hand-written migration. Nothing in the running application compares the two — + * the next `npm run migration` does, and an entity that has drifted from the migration produces + * a migration "fixing" the difference in the direction of the entity. + */ + it('is declared on the entity under the name the migration creates', () => { + const declared = getMetadataArgsStorage().indices.filter((index) => index.target === PaymentLinkPayment); + + expect(declared.map((index) => index.columns)).toContainEqual(['deviceId']); + + const name = new DefaultNamingStrategy().indexName('payment_link_payment', ['deviceId']); + + expect(readFileSync(MIGRATION, 'utf8')).toContain( + `CREATE INDEX "${name}" ON "payment_link_payment" ("deviceId")`, + ); + }); + }); +}); diff --git a/src/subdomains/core/payment-link/entities/payment-link-payment.entity.ts b/src/subdomains/core/payment-link/entities/payment-link-payment.entity.ts index c16fe33ebb..ef434cbe0a 100644 --- a/src/subdomains/core/payment-link/entities/payment-link-payment.entity.ts +++ b/src/subdomains/core/payment-link/entities/payment-link-payment.entity.ts @@ -49,6 +49,7 @@ export class PaymentLinkPayment extends IEntity { @Column({ default: false }) isConfirmed: boolean; + @Index() @Column({ length: 256, nullable: true }) deviceId?: string; @@ -99,4 +100,18 @@ export class PaymentLinkPayment extends IEntity { get device(): PaymentDevice | undefined { return this.deviceId && this.deviceCommand ? { id: this.deviceId, command: this.deviceCommand } : undefined; } + + /** + * The persisted state a caller of `PaymentLinkPaymentService.waitForPayment` is released on. + * + * `status` alone is not enough: a `MULTIPLE`-mode payment stays `Pending` while its quotes + * complete one after another, and each of those releases the callers waiting at that moment + * (see `PaymentLinkPaymentService.handleQuoteChange`). What changes there is `txCount`. + * + * It is therefore a value to compare against the one the payment carried when the wait started, + * not a predicate: for a `MULTIPLE`-mode payment there is no absolute "released" state to test. + */ + get waitState(): string { + return `${this.status}:${this.txCount}`; + } } diff --git a/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts b/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts new file mode 100644 index 0000000000..6472975ad2 --- /dev/null +++ b/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts @@ -0,0 +1,203 @@ +import { BlockchainRegistryService } from 'src/integration/blockchain/shared/services/blockchain-registry.service'; +import { FiatService } from 'src/shared/models/fiat/fiat.service'; +import { In } from 'typeorm'; +import { PaymentDevice, PaymentLinkPayment } from '../../entities/payment-link-payment.entity'; +import { PaymentLinkPaymentMode, PaymentLinkPaymentStatus } from '../../enums'; +import { PaymentLinkPaymentRepository } from '../../repositories/payment-link-payment.repository'; +import { PaymentActivationService } from '../payment-activation.service'; +import { PaymentLinkPaymentService } from '../payment-link-payment.service'; +import { PaymentQuoteService } from '../payment-quote.service'; +import { PaymentWebhookService } from '../payment-webhook.service'; + +/** + * The delivery channels of this service are process-local, while the jobs writing the payments run + * in one process only. Every test here therefore describes one process: what it holds, what it may + * read, and what it delivers — never who wrote the row it reads. + */ +describe('PaymentLinkPaymentService', () => { + let service: PaymentLinkPaymentService; + let paymentLinkPaymentRepo: jest.Mocked; + let paymentWebhookService: jest.Mocked; + let paymentQuoteService: jest.Mocked; + let paymentActivationService: jest.Mocked; + + function payment(values: Partial): PaymentLinkPayment { + return Object.assign(new PaymentLinkPayment(), { + id: 7, + status: PaymentLinkPaymentStatus.PENDING, + mode: PaymentLinkPaymentMode.SINGLE, + txCount: 0, + updated: new Date('2026-01-01T10:00:00Z'), + link: {}, + ...values, + }); + } + + /** Resolves to the payment if it was delivered, and to a marker if nothing was. */ + async function delivery(waiting: Promise): Promise { + return Promise.race([ + waiting, + new Promise((resolve) => setTimeout(() => resolve('nothing delivered'), 50)), + ]); + } + + function devices(): PaymentDevice[] { + const seen: PaymentDevice[] = []; + service.getDeviceActivationObservable().subscribe((device) => seen.push(device)); + + return seen; + } + + beforeEach(() => { + paymentLinkPaymentRepo = { + find: jest.fn().mockResolvedValue([]), + save: jest.fn().mockImplementation((entity) => entity), + } as unknown as jest.Mocked; + + paymentWebhookService = { sendWebhook: jest.fn() } as unknown as jest.Mocked; + + paymentQuoteService = { cancelAllForPayment: jest.fn() } as unknown as jest.Mocked; + + paymentActivationService = { closeAllForPayment: jest.fn() } as unknown as jest.Mocked; + + service = new PaymentLinkPaymentService( + {} as unknown as jest.Mocked, + paymentLinkPaymentRepo, + paymentWebhookService, + paymentQuoteService, + paymentActivationService, + {} as unknown as jest.Mocked, + ); + }); + + // --- deliverPaymentUpdates() Tests --- // + + describe('deliverPaymentUpdates()', () => { + it('should release a caller waiting here on a payment another process wrote', async () => { + const waiting = service.waitForPayment(payment({ id: 7 })); + + // Nothing in this process wrote the payment, so nothing in this process released the caller. + expect(await delivery(waiting)).toEqual('nothing delivered'); + + paymentLinkPaymentRepo.find.mockResolvedValue([payment({ id: 7, status: PaymentLinkPaymentStatus.COMPLETED })]); + await service.deliverPaymentUpdates(); + + expect(await delivery(waiting)).toMatchObject({ id: 7, status: PaymentLinkPaymentStatus.COMPLETED }); + }); + + it('should release a caller on a MULTIPLE-mode payment that stays pending', async () => { + const waiting = service.waitForPayment(payment({ id: 7, mode: PaymentLinkPaymentMode.MULTIPLE, txCount: 1 })); + + paymentLinkPaymentRepo.find.mockResolvedValue([ + payment({ id: 7, mode: PaymentLinkPaymentMode.MULTIPLE, txCount: 2 }), + ]); + await service.deliverPaymentUpdates(); + + expect(await delivery(waiting)).toMatchObject({ id: 7, txCount: 2 }); + }); + + it('should keep waiting while the payment is unchanged', async () => { + const waiting = service.waitForPayment(payment({ id: 7 })); + + paymentLinkPaymentRepo.find.mockResolvedValue([payment({ id: 7 })]); + await service.deliverPaymentUpdates(); + + expect(await delivery(waiting)).toEqual('nothing delivered'); + }); + + it('should not touch the database while this process holds neither a caller nor a device', async () => { + await service.deliverPaymentUpdates(); + + expect(paymentLinkPaymentRepo.find).not.toHaveBeenCalled(); + }); + + it('should ask only for the payments this process is waiting on', async () => { + void service.waitForPayment(payment({ id: 7 })); + + await service.deliverPaymentUpdates(); + + expect(paymentLinkPaymentRepo.find).toHaveBeenCalledTimes(1); + expect(paymentLinkPaymentRepo.find).toHaveBeenCalledWith(expect.objectContaining({ where: { id: In([7]) } })); + }); + + it('should send the command to a device connected here after another process wrote the payment', async () => { + const seen = devices(); + service.registerDevice('pos-1'); + + paymentLinkPaymentRepo.find.mockResolvedValue([ + payment({ + id: 7, + status: PaymentLinkPaymentStatus.COMPLETED, + deviceId: 'pos-1', + deviceCommand: 'show-paid', + updated: new Date('2026-01-01T11:00:00Z'), + }), + ]); + await service.deliverPaymentUpdates(); + + expect(seen).toEqual([{ id: 'pos-1', command: 'show-paid' }]); + }); + + it('should send the same payment state to a device once', async () => { + const seen = devices(); + service.registerDevice('pos-1'); + + paymentLinkPaymentRepo.find.mockResolvedValue([ + payment({ + id: 7, + status: PaymentLinkPaymentStatus.COMPLETED, + deviceId: 'pos-1', + deviceCommand: 'show-paid', + updated: new Date('2026-01-01T11:00:00Z'), + }), + ]); + await service.deliverPaymentUpdates(); + await service.deliverPaymentUpdates(); + + expect(seen).toHaveLength(1); + }); + + it('should stop looking for a device whose last connection closed', async () => { + service.registerDevice('pos-1'); + service.registerDevice('pos-1'); + service.unregisterDevice('pos-1'); + + await service.deliverPaymentUpdates(); + expect(paymentLinkPaymentRepo.find).toHaveBeenCalledTimes(1); + + service.unregisterDevice('pos-1'); + + await service.deliverPaymentUpdates(); + expect(paymentLinkPaymentRepo.find).toHaveBeenCalledTimes(1); + }); + }); + + // --- doSave() Tests --- // + + describe('doSave()', () => { + it('should release a caller in the writing process without waiting for the job', async () => { + const pending = payment({ id: 7, deviceId: 'pos-1', deviceCommand: 'show-paid' }); + const seen = devices(); + service.registerDevice('pos-1'); + + const waiting = service.waitForPayment(pending); + await service.expirePayment(pending); + + expect(await delivery(waiting)).toMatchObject({ id: 7, status: PaymentLinkPaymentStatus.EXPIRED }); + expect(seen).toEqual([{ id: 'pos-1', command: 'show-paid' }]); + }); + + it('should not repeat a delivery the writing process already made', async () => { + const pending = payment({ id: 7, deviceId: 'pos-1', deviceCommand: 'show-paid' }); + const seen = devices(); + service.registerDevice('pos-1'); + + await service.expirePayment(pending); + + paymentLinkPaymentRepo.find.mockResolvedValue([pending]); + await service.deliverPaymentUpdates(); + + expect(seen).toHaveLength(1); + }); + }); +}); diff --git a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts index e7f223f2d9..a2358e6dc5 100644 --- a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts @@ -10,7 +10,7 @@ import { AsyncMap } from 'src/shared/utils/async-map'; import { Util } from 'src/shared/utils/util'; import { C2BWebhookResult } from 'src/subdomains/core/payment-link/share/c2b-payment-link.provider'; import { CryptoInput } from 'src/subdomains/supporting/payin/entities/crypto-input.entity'; -import { IsNull, LessThan } from 'typeorm'; +import { In, IsNull, LessThan, MoreThan, Not } from 'typeorm'; import { isSellRoute } from '../../sell-crypto/route/sell.entity'; import { CreatePaymentLinkPaymentDto } from '../dto/create-payment-link-payment.dto'; import { PaymentLinkEvmPaymentDto, PaymentLinkHexResultDto, TransferInfo } from '../dto/payment-link.dto'; @@ -33,10 +33,22 @@ import { PaymentActivationService } from './payment-activation.service'; import { PaymentQuoteService } from './payment-quote.service'; import { PaymentWebhookService } from './payment-webhook.service'; +/** What one process knows about the websocket connections it holds open for a single device. */ +interface DeviceSubscription { + /** Open connections for this device on this process. The entry is dropped when the last closes. */ + connections: number; + /** Payments updated at or before this are not delivered: they predate the oldest connection. */ + since: Date; + /** `:` of the last command delivered, so the same state is sent once. */ + delivered?: string; +} + @Injectable() export class PaymentLinkPaymentService { private readonly paymentWaitMap = new AsyncMap(this.constructor.name); + private readonly waitStates = new Map(); private readonly deviceActivationSubject = new Subject(); + private readonly deviceSubscriptions = new Map(); constructor( private readonly fiatService: FiatService, @@ -163,10 +175,124 @@ export class PaymentLinkPaymentService { } // --- HANDLE WAITS --- // + + /** + * Both delivery channels of this service are process-local: the map behind this method and the + * subject behind `getDeviceActivationObservable`. A caller therefore only ever hears from the + * process holding its connection, while the jobs that move a payment forward run in one process + * (`CronScope.WORKER`, held down to a single process by the cron lease). + * + * `deliverPaymentUpdates` below bridges the two. It reads the persisted state of the payments + * THIS process is waiting on and releases them here, so the delivery no longer depends on which + * process did the writing. `doSave` still delivers directly, which keeps the single-process + * case (`CRON_ROLE=all`) as immediate as it is today; the job is the catch-up path for every + * other process. + */ async waitForPayment(payment: PaymentLinkPayment): Promise { + // The state to compare against, taken before the wait: what the caller is waiting for is a + // change from it, not a fixed target state (see PaymentLinkPayment.waitState). + if (!this.waitStates.has(payment.id)) this.waitStates.set(payment.id, payment.waitState); + return this.paymentWaitMap.wait(payment.id, 0); } + /** Called by PaymentLinkGateway for every websocket connection it accepts. */ + registerDevice(deviceId: string): void { + const subscription = this.deviceSubscriptions.get(deviceId); + + if (subscription) { + subscription.connections++; + } else { + this.deviceSubscriptions.set(deviceId, { connections: 1, since: new Date() }); + } + } + + /** Called by PaymentLinkGateway when one of those connections closes. */ + unregisterDevice(deviceId: string): void { + const subscription = this.deviceSubscriptions.get(deviceId); + if (!subscription) return; + + subscription.connections--; + if (subscription.connections < 1) this.deviceSubscriptions.delete(deviceId); + } + + /** + * Delivers what this process is waiting for, from the database rather than from the job that + * wrote it. Writes nothing and calls nothing outside the process, so it is safe to run in every + * process at once — which is what `CronScope.BOTH` requires and what makes it exempt from the + * lease that confines the writing jobs to one process. + * + * Both halves are bounded by what this process actually holds: with no caller waiting and no + * device connected they touch the database not at all. + */ + async deliverPaymentUpdates(): Promise { + await this.deliverToWaitingCallers(); + await this.deliverToConnectedDevices(); + } + + private async deliverToWaitingCallers(): Promise { + const ids = this.paymentWaitMap.get(); + if (!ids.length) return; + + const payments = await this.paymentLinkPaymentRepo.find({ + where: { id: In(ids) }, + relations: { link: true }, + }); + + for (const payment of payments) { + if (this.waitStates.get(payment.id) !== payment.waitState) this.resolveWaiters(payment); + } + } + + private async deliverToConnectedDevices(): Promise { + const subscriptions = Array.from(this.deviceSubscriptions.entries()); + if (!subscriptions.length) return; + + const deviceIds = subscriptions.map(([deviceId]) => deviceId); + const since = Util.minObj( + subscriptions.map(([, subscription]) => subscription), + 'since', + ).since; + + // The same condition the direct delivery in doSave runs under, expressed over stored columns: + // a payment out of `Pending`, or a `MULTIPLE`-mode payment that has counted a completed quote. + const payments = await this.paymentLinkPaymentRepo.find({ + where: [ + { deviceId: In(deviceIds), updated: MoreThan(since), status: Not(PaymentLinkPaymentStatus.PENDING) }, + { deviceId: In(deviceIds), updated: MoreThan(since), txCount: MoreThan(0) }, + ], + order: { updated: 'ASC' }, + }); + + for (const payment of payments) this.deliverToDevice(payment); + } + + private resolveWaiters(payment: PaymentLinkPayment): void { + this.waitStates.delete(payment.id); + this.paymentWaitMap.resolve(payment.id, payment); + } + + /** + * Idempotent by the state it delivers: a device is sent the same command for the same payment + * state once, whether this process wrote it or read it back. Both callers go through here. + */ + private deliverToDevice(payment: PaymentLinkPayment): void { + const device = payment.device; + if (!device) return; + + const subscription = this.deviceSubscriptions.get(device.id); + if (!subscription) return; + + const state = `${payment.id}:${payment.waitState}`; + if (state === subscription.delivered) return; + + subscription.delivered = state; + // Keeps the window of the query above from growing over the lifetime of a connection. + if (payment.updated > subscription.since) subscription.since = payment.updated; + + this.deviceActivationSubject.next(device); + } + async handleBinanceWaiting(result: C2BWebhookResult): Promise { const { qrContent, referId } = result.metadata; @@ -437,9 +563,12 @@ export class PaymentLinkPaymentService { if (savedPayment.link.webhookUrl) await this.sendWebhook(savedPayment); + // Delivers to this process directly, which is the whole latency budget when the writing job + // and the waiting caller share a process. Whoever waits elsewhere is served by + // deliverPaymentUpdates, which reads the row this save just wrote. if (isPaymentDone) { - this.paymentWaitMap.resolve(savedPayment.id, savedPayment); - if (payment.device) this.deviceActivationSubject.next(payment.device); + this.resolveWaiters(savedPayment); + this.deliverToDevice(savedPayment); } return savedPayment; From ef959012b352adac5fd20838c0c2751917aec80c Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:13 +0200 Subject: [PATCH 47/86] Run the payment jobs that write in one process and the delivery in every one processExpiredPayments and checkTxConfirmations write to the database, trigger merchant webhooks and cancel quotes. That is worker work, and the lease keeps it to a single process. They were scoped api because they also release local waiters, which is now done by deliverPaymentUpdates instead - scope both, exempt from the lease, and running every 15 seconds so it does not add a second minute to a chain that already costs one. --- docs/cron-jobs.md | 18 ++++--- src/shared/services/process.service.ts | 1 + .../__tests__/payment-cron.service.spec.ts | 27 ++++++++++ .../services/payment-cron.service.ts | 53 ++++++++++++------- 4 files changed, 71 insertions(+), 28 deletions(-) create mode 100644 src/subdomains/core/payment-link/services/__tests__/payment-cron.service.spec.ts diff --git a/docs/cron-jobs.md b/docs/cron-jobs.md index ff8acc2895..e8600ba637 100644 --- a/docs/cron-jobs.md +++ b/docs/cron-jobs.md @@ -1,6 +1,6 @@ # Cron jobs -Every scheduled job this service runs: **138 `@DfxCron` declarations** across 97 files and 34 areas. +Every scheduled job this service runs: **139 `@DfxCron` declarations** across 97 files and 34 areas. ## Columns @@ -15,7 +15,7 @@ Every scheduled job this service runs: **138 `@DfxCron` declarations** across 97 ## Scopes `scope` is a mandatory parameter of `@DfxCron` and says which process registers the job: -117 are `worker`, 7 are `api`, 14 are `both`. `CRON_ROLE` decides what a process is +119 are `worker`, 5 are `api`, 15 are `both`. `CRON_ROLE` decides what a process is (`worker`, `api`, or `all` for a single-process setup); a process runs its own scope plus `both`. `worker` is the normal case — anything writing to the database or driving business forward belongs @@ -30,7 +30,7 @@ request path loads on demand, and a job may refresh it but must not be the only ## Flags -117 of the 138 jobs carry a `process` flag, 21 do not. A job with a flag can be switched off +118 of the 139 jobs carry a `process` flag, 21 do not. A job with a flag can be switched off without a deploy — `DfxCronService` skips it when the process appears in the disabled set, which `ProcessService` refreshes from the `disabledProcesses` setting and the `DISABLED_PROCESSES` environment variable every 30 seconds. @@ -62,6 +62,7 @@ New jobs should declare a flag unless there is a reason like the one above. | -------- | ---: | | second | 5 | | 10 seconds | 3 | +| 15 seconds | 1 | | 30 seconds | 9 | | minute | 52 | | 5 minutes | 18 | @@ -90,7 +91,7 @@ Jobs by area: | `shared/services` | 5 | 5 | | `subdomains/core/sell-crypto` | 5 | 2 | | `subdomains/supporting/payment` | 5 | 1 | -| `subdomains/core/payment-link` | 4 | — | +| `subdomains/core/payment-link` | 5 | — | | `subdomains/generic/kyc` | 4 | — | | `subdomains/supporting/bank` | 4 | — | | `subdomains/supporting/bank-tx` | 4 | — | @@ -121,7 +122,7 @@ Jobs by area: Every `@DfxCron(` occurrence in `src/**/*.ts`. Decorator arguments are read by a balanced-paren scan, so multi-line declarations are included — a line-based match misses 26 of them. Interval, flag and scope come from those arguments, so all three are as accurate as the source. The parsed -count is asserted against a raw text count of the decorator: **138 = 138**, no gap. Class and +count is asserted against a raw text count of the decorator: **139 = 139**, no gap. Class and method come from the enclosing `export class` (including `export abstract class`) and the identifier following the decorator. @@ -145,7 +146,7 @@ the job is registered — on the provider instance, which is a different object instance the request handlers use. Resolving either one is a decision about the jobs, not about this inventory, so both are recorded -here rather than fixed in passing. Of the 138 declarations, 137 have a registration path. +here rather than fixed in passing. Of the 139 declarations, 138 have a registration path. ## Jobs @@ -159,6 +160,7 @@ here rather than fixed in passing. Of the 138 declarations, 137 have a registrat | 10 seconds | `LIQUIDITY_MANAGEMENT` | `worker` | `LiquidityManagementPipelineService::processPipelines` | `subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.ts` | | 10 seconds | `MONITOR_CONNECTION_POOL` | `both` | `MonitorConnectionPoolService::monitorConnectionPoolStatic` | `subdomains/core/monitoring/monitor-connection-pool.service.ts` | | 10 seconds | `MONITOR_EVENT_LOOP` | `both` | `MonitorEventLoopService::monitorEventLoop` | `subdomains/core/monitoring/monitor-event-loop.service.ts` | +| 15 seconds | `PAYMENT_DELIVERY` | `both` | `PaymentCronService::deliverPaymentUpdates` | `subdomains/core/payment-link/services/payment-cron.service.ts` | | 30 seconds | `LNURL_AUTH_CACHE` | `both` | `AuthLnUrlService::processCleanupAccessToken` | `subdomains/generic/user/models/auth/auth-lnurl.service.ts` | | 30 seconds | `BANK_TX` | `worker` | `BankTxService::checkBankTx` | `subdomains/supporting/bank-tx/bank-tx/services/bank-tx.service.ts` | | 30 seconds | `DEX_PURCHASE_ORDER` | `worker` | `DexService::finalizePurchaseOrders` | `subdomains/supporting/dex/services/dex.service.ts` | @@ -205,8 +207,8 @@ here rather than fixed in passing. Of the 138 declarations, 137 have a registrat | minute | `PAY_IN` | `worker` | `PayInService::checkConfirmations` | `subdomains/supporting/payin/services/payin.service.ts` | | minute | `PAY_IN` | `worker` | `PayInService::forwardPayInEntries` | `subdomains/supporting/payin/services/payin.service.ts` | | minute | `PAY_IN` | `worker` | `PayInService::returnPayInEntries` | `subdomains/supporting/payin/services/payin.service.ts` | -| minute | `PAYMENT_CONFIRMATIONS` | `api` | `PaymentCronService::checkTxConfirmations` | `subdomains/core/payment-link/services/payment-cron.service.ts` | -| minute | `PAYMENT_EXPIRATION` | `api` | `PaymentCronService::processExpiredPayments` | `subdomains/core/payment-link/services/payment-cron.service.ts` | +| minute | `PAYMENT_CONFIRMATIONS` | `worker` | `PaymentCronService::checkTxConfirmations` | `subdomains/core/payment-link/services/payment-cron.service.ts` | +| minute | `PAYMENT_EXPIRATION` | `worker` | `PaymentCronService::processExpiredPayments` | `subdomains/core/payment-link/services/payment-cron.service.ts` | | minute | `UPDATE_BLOCKCHAIN_FEE` | `api` | `PaymentLinkFeeService::updateFees` | `subdomains/core/payment-link/services/payment-link-fee.service.ts` | | minute | `REALUNIT_QUOTE_COMPLETION` | `worker` | `RealUnitJobService::completeSettledQuotes` | `subdomains/supporting/realunit/realunit-job.service.ts` | | minute | — | `worker` | `StaffKycClearanceService::syncStaffKycClearance` | `subdomains/generic/user/models/user/staff-kyc-clearance.service.ts` | diff --git a/src/shared/services/process.service.ts b/src/shared/services/process.service.ts index d00d31deaa..d262fb45f3 100644 --- a/src/shared/services/process.service.ts +++ b/src/shared/services/process.service.ts @@ -62,6 +62,7 @@ export enum Process { SCORECHAIN = 'Scorechain', PAYMENT_EXPIRATION = 'PaymentExpiration', PAYMENT_CONFIRMATIONS = 'PaymentConfirmations', + PAYMENT_DELIVERY = 'PaymentDelivery', PAYMENT_FORWARDING = 'PaymentForwarding', FIAT_OUTPUT = 'FiatOutput', FIAT_OUTPUT_ASSIGN_BANK_ACCOUNT = 'FiatOutputAssignBankAccount', diff --git a/src/subdomains/core/payment-link/services/__tests__/payment-cron.service.spec.ts b/src/subdomains/core/payment-link/services/__tests__/payment-cron.service.spec.ts new file mode 100644 index 0000000000..0884cf1e65 --- /dev/null +++ b/src/subdomains/core/payment-link/services/__tests__/payment-cron.service.spec.ts @@ -0,0 +1,27 @@ +import { DFX_CRONJOB_PARAMS, CronScope, DfxCronParams } from 'src/shared/utils/cron'; +import { PaymentCronService } from '../payment-cron.service'; + +/** + * The scopes of these three jobs are the whole point of the split between writing and delivering, + * and nothing at runtime notices when one of them changes: a writing job scoped `both` would + * duplicate its webhooks, and the delivery job scoped `worker` or `api` would take the lease and + * leave the callers of every other process unreleased. Both are silent. + */ +describe('PaymentCronService', () => { + function scopeOf(method: keyof PaymentCronService): CronScope { + const params: DfxCronParams = Reflect.getMetadata(DFX_CRONJOB_PARAMS, PaymentCronService.prototype[method]); + + return params?.scope; + } + + it.each(['processExpiredPayments', 'checkTxConfirmations', 'forwardDeposits'] as const)( + 'runs %s in one process, because it writes and calls out', + (method) => { + expect(scopeOf(method)).toEqual(CronScope.WORKER); + }, + ); + + it('runs the delivery in every process, because each holds its own callers and devices', () => { + expect(scopeOf('deliverPaymentUpdates')).toEqual(CronScope.BOTH); + }); +}); diff --git a/src/subdomains/core/payment-link/services/payment-cron.service.ts b/src/subdomains/core/payment-link/services/payment-cron.service.ts index 31e4b10ba4..79d63cc5f6 100644 --- a/src/subdomains/core/payment-link/services/payment-cron.service.ts +++ b/src/subdomains/core/payment-link/services/payment-cron.service.ts @@ -2,6 +2,7 @@ import { Injectable } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; import { Process } from 'src/shared/services/process.service'; import { CronScope, DfxCron } from 'src/shared/utils/cron'; +import { CustomCronExpression } from 'src/shared/utils/custom-cron-expression'; import { PaymentActivationService } from './payment-activation.service'; import { PaymentBalanceService } from './payment-balance.service'; import { PaymentLinkPaymentService } from './payment-link-payment.service'; @@ -16,37 +17,49 @@ export class PaymentCronService { private readonly paymentBalanceService: PaymentBalanceService, ) {} - // Api, not Worker: this job and checkTxConfirmations both end up in - // PaymentLinkPaymentService.doSave(), which resolves the AsyncMap that PaymentLinkController's - // waitForPayment awaits and pushes the device activation into the RxJS subject PaymentLinkGateway - // subscribes to. Both hold their state in the instance, so they only reach a caller of this same - // process. `Both` is no option either: the jobs write to the database and trigger merchant - // webhooks, which a second registration would repeat. + // The three jobs below split what used to be one decision. Writing and delivering have opposite + // requirements — a database write, a merchant webhook and a quote cancellation must happen once + // in the deployment, while the AsyncMap and the RxJS subject in PaymentLinkPaymentService are + // process-local and only reach a caller connected to the process that fires them. A single scope + // cannot satisfy both: `Worker` or `Api` leaves callers on every other process unreleased, + // `Both` repeats every write and every webhook. // - // Those two properties pull against each other, and the scope alone cannot settle it. "Only - // reaches a caller of this process" argues for running in every API process; "writes and calls - // out" argues for running in exactly one. DfxCronService leases these jobs, so it is the second: - // with more than one API process, the one that loses the lease does not release the callers - // waiting on it, and they wait until their own timeout. That is the recoverable side — a - // duplicated payout or merchant webhook is not. The lease reports the lost race at error level - // for exactly this reason; see DfxCronService.guardAcrossProcesses. - // - // Resolving it properly means driving the local delivery from persisted state rather than from - // the job that does the writing, so any process can release its own waiters. That is a change to - // PaymentLinkPaymentService, not to this scope. - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.API, process: Process.PAYMENT_EXPIRATION }) + // So the writing runs under the lease (`Worker`), and deliverPaymentUpdates delivers from the + // persisted state those writes leave behind, in every process, without a lease. It writes + // nothing and calls nothing outside its process, which is what allows it to run everywhere. + + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.PAYMENT_EXPIRATION }) async processExpiredPayments(): Promise { await this.paymentLinkPaymentService.processExpiredPayments(); await this.paymentActivationService.processExpiredActivations(); await this.paymentQuoteService.processExpiredQuotes(); } - // Api for the same reason as processExpiredPayments above. - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.API, process: Process.PAYMENT_CONFIRMATIONS }) + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.PAYMENT_CONFIRMATIONS }) async checkTxConfirmations(): Promise { await this.paymentLinkPaymentService.checkTxConfirmations(); } + // Runs at 15 seconds rather than the minute the two jobs above run at, because it is the second + // hop of a chain: the writing job already costs up to a minute to notice, and this must not add + // another one to it. It stays cheap at that rate by looking only at what its own process holds — + // with no caller waiting and no device connected it issues no query at all. + // + // `useDelay: false` for the same reason: the jitter exists to spread jobs that do real work per + // run, and up to five seconds of it would be a third of this interval. + // + // Switching the flag off does not stop payments from being processed, but callers of + // `GET /v1/paymentLink/payment/wait` and `GET /v1/lnurlp/wait/:id` on a process that is not the + // one writing then stay connected until they give up. + @DfxCron(CustomCronExpression.EVERY_15_SECONDS, { + scope: CronScope.BOTH, + process: Process.PAYMENT_DELIVERY, + useDelay: false, + }) + async deliverPaymentUpdates(): Promise { + await this.paymentLinkPaymentService.deliverPaymentUpdates(); + } + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.WORKER, process: Process.PAYMENT_FORWARDING }) async forwardDeposits(): Promise { await this.paymentBalanceService.forwardDeposits(); From bead4ed6f5107cd6591747e6bf592882837eeb47 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:15 +0200 Subject: [PATCH 48/86] Say what the api scope is for now that the delivery no longer sits in it No api-scoped job drives work bound to the connections its process holds open any more, and none of them may: the lease confines the job to one process while the connections are spread over all of them. The scope doc, the lease comment and both documents said otherwise. --- CONTRIBUTING.md | 5 +++-- docs/cron-jobs.md | 3 ++- src/shared/services/dfx-cron.service.ts | 17 +++++++++-------- src/shared/utils/cron.ts | 8 ++++++-- .../services/payment-link-payment.service.ts | 7 +++++++ 5 files changed, 27 insertions(+), 13 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d19cf14bbb..e66201a9be 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -568,8 +568,9 @@ async processPayments(): Promise {} @DfxCron(CronExpression.EVERY_30_SECONDS, { scope: CronScope.BOTH }) async resyncDeniedJwtAccounts(): Promise {} -// Api: maintains state read only from a request path, or drives work bound to the connections -// this process holds open. +// Api: maintains state read only from a request path. Not for delivering to the connections +// this process holds open — that job is leased too, so it would run in one process while the +// connections are spread over all of them. Deliver from stored state under `Both` instead. @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.API, process: Process.UPDATE_STATISTIC }) async doUpdate(): Promise {} ``` diff --git a/docs/cron-jobs.md b/docs/cron-jobs.md index e8600ba637..032b093c16 100644 --- a/docs/cron-jobs.md +++ b/docs/cron-jobs.md @@ -22,7 +22,8 @@ Every scheduled job this service runs: **139 `@DfxCron` declarations** across 97 to exactly one process. `both` is for a job maintaining process-local state that a request path also reads, so it must run everywhere; running it twice has to be harmless by construction, which rules out database writes, mail and paid external calls. `api` is for state read only from a -request path, or for work bound to the connections that process holds open. +request path — not for delivering to the connections a process holds open, because an `api` job +is leased and would run in one process while the connections are spread over all of them. Getting the scope wrong fails silently: the cache a job maintains simply stays empty in the process that reads it. The rule that keeps that harmless is in CONTRIBUTING.md — a cache read in a diff --git a/src/shared/services/dfx-cron.service.ts b/src/shared/services/dfx-cron.service.ts index 45cdb8e66c..2773eb40cd 100644 --- a/src/shared/services/dfx-cron.service.ts +++ b/src/shared/services/dfx-cron.service.ts @@ -162,14 +162,15 @@ export class DfxCronService implements OnModuleInit { * state. Their safety comes from a different property: running twice must be harmless by * construction, which is what CONTRIBUTING requires of them. * - * `API` jobs are leased, and reported when they lose the race. They are declared `API` because - * their effect is confined to the process running them, which is an argument for running them - * in every API process; they also write to the database and call out, which is an argument for - * running them once. Only one of those can be had. The lease is the side that cannot be undone - * afterwards — a duplicated payout or webhook stays duplicated, whereas a caller that was not - * released is a request that times out. Losing the race therefore stays possible by design, and - * is reported rather than passed over, because for these jobs it is the symptom of a deployment - * running more than one API process rather than a normal cycle. + * `API` jobs are leased as well, and reported when they lose the race. Their scope says their + * effect is confined to the process running them, which is an argument for every API process + * running them; the lease is what keeps one that also writes or calls out from doing either + * twice. Where the two pull apart the lease wins, and the process that lost the race is left + * without whatever the job maintains — which is why an `API` job must not be the only thing + * filling what a request path reads, and why delivering to connections belongs to a `BOTH` job + * driven from stored state rather than to this scope. Losing the race is reported rather than + * passed over, because for these jobs it is the symptom of a deployment running more than one + * API process rather than a normal cycle. */ private guardAcrossProcesses(cronJobName: string, data: CronJobData): () => Promise { const task = this.wrapFunction(data); diff --git a/src/shared/utils/cron.ts b/src/shared/utils/cron.ts index ceda9f8fdc..08c76d2dd8 100644 --- a/src/shared/utils/cron.ts +++ b/src/shared/utils/cron.ts @@ -15,8 +15,12 @@ export enum CronScope { /** Worker process only. The normal case: anything writing to the database or driving business forward. */ WORKER = 'worker', /** - * API process only. Maintains or measures state read exclusively from a request path, or - * drives work bound to the connections that process holds open. + * API process only. Maintains or measures state read exclusively from a request path. + * + * Not for delivering to the connections a process holds open: such a job is leased like any + * other, so it would run in one process while the connections are spread over all of them. + * Delivery is driven from stored state and scoped `BOTH` - see + * `PaymentLinkPaymentService.deliverPaymentUpdates`. */ API = 'api', /** diff --git a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts index a2358e6dc5..221c88ad07 100644 --- a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts @@ -372,6 +372,13 @@ export class PaymentLinkPaymentService { } // expiry timers + // + // These stay in the process that served the request, although processExpiredPayments is a + // worker job. Moving the job did not give a payment a second timer: a timer is armed once, by + // the process that created the payment, and expirePaymentIfPending re-reads the payment with + // `status: Pending`, so a payment the job has already expired is left alone here. What the + // timers buy is what they bought before — a caller waiting on this process is released at the + // timeout rather than at the next tick of a job elsewhere. const scanTimeout = paymentLink.configObj.scanTimeout; if (scanTimeout) { setTimeout(() => this.expirePaymentIfPending(payment.id, true), scanTimeout * 1000); From 12b901ae49c7ae0e8d092fea9684842c149c49bc Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:16 +0200 Subject: [PATCH 49/86] Let every lease operation clear the health flag, and stop overlapping renewals A successful renew now reports the lease layer as usable, not only a successful acquire: the heartbeat reports this as a state, and a process running long jobs renews for minutes without claiming anything new. The renewal re-arms only once the previous attempt has settled, so a slow database cannot pile up attempts that each hold a pooled connection. Shutdown now closes the lease to new runs before it takes its snapshot of the running ones. A run started after that snapshot was waited for by nobody and was cut off part-way through when the process exited. --- .../__tests__/cron-lease.service.spec.ts | 99 ++++++++++++++++++- src/shared/services/cron-lease.service.ts | 94 +++++++++++++++--- 2 files changed, 176 insertions(+), 17 deletions(-) diff --git a/src/shared/services/__tests__/cron-lease.service.spec.ts b/src/shared/services/__tests__/cron-lease.service.spec.ts index 1dd7d1c6a1..f39c02a22d 100644 --- a/src/shared/services/__tests__/cron-lease.service.spec.ts +++ b/src/shared/services/__tests__/cron-lease.service.spec.ts @@ -26,6 +26,9 @@ describe('CronLeaseService', () => { return { service: new CronLeaseService(createMock({ query: onQuery })), onQuery }; } + /** Lets pending promises settle without advancing any timer. */ + const settle = () => new Promise((resolve) => setImmediate(resolve)); + beforeEach(() => { process.env.CRON_ROLE = 'worker'; new ConfigService(GetConfig()); @@ -150,6 +153,44 @@ describe('CronLeaseService', () => { expect(onQuery.mock.calls[0][1][2]).toEqual('60'); }); + + it('keeps one renewal outstanding at a time', async () => { + // A fixed interval fires whether or not the previous renewal came back. A database that + // answers slowly is exactly when this matters: the attempts pile up, each holding a pooled + // connection, and an older answer can land after a newer one. Here the renewal never comes + // back at all, so a fixed interval would have started two more by the time this asserts. + jest.useFakeTimers({ doNotFake: ['setImmediate', 'nextTick'] }); + + try { + let renewals = 0; + const onQuery = jest.fn().mockImplementation((sql: string) => { + if (sql.includes('INSERT INTO')) return Promise.resolve([{ owner: 'worker:1' }]); + if (sql.includes('UPDATE')) { + renewals++; + return new Promise(() => undefined); + } + + return Promise.resolve([]); + }); + + const { service } = buildService({ onQuery }); + let finish: () => void; + const run = service.run('SomeService::job', () => new Promise((resolve) => (finish = resolve))); + + await settle(); + + // Three renewal intervals (20 s each) with the first one still unanswered. + jest.advanceTimersByTime(61_000); + await settle(); + + expect(renewals).toEqual(1); + + finish(); + await run; + } finally { + jest.useRealTimers(); + } + }); }); describe('shutdown', () => { @@ -157,9 +198,6 @@ describe('CronLeaseService', () => { // successor can pick the job up, and whether it can pick it up while this process still works // on it. - /** Lets pending promises settle without advancing any timer. */ - const settle = () => new Promise((resolve) => setImmediate(resolve)); - const released = (onQuery: jest.Mock) => onQuery.mock.calls.some(([sql]) => (sql as string).includes('DELETE FROM')); @@ -252,6 +290,44 @@ describe('CronLeaseService', () => { await service.shutdown(); }); + it('starts no further job once shutdown has begun', async () => { + // The wait above covers the jobs that were running when shutdown took its snapshot, and the + // process exits when that wait ends. A job started afterwards is in no snapshot, so it would + // be cut off part-way through — with the exit landing before its own `finally`. + const { service, onQuery } = buildService({}); + const task = jest.fn().mockResolvedValue(undefined); + + await service.shutdown(); + await service.run('SomeService::job', task); + + expect(task).not.toHaveBeenCalled(); + expect(onQuery.mock.calls.some(([sql]) => (sql as string).includes('INSERT INTO'))).toBe(false); + }); + + it('hands the claim back when shutdown begins while it is being taken', async () => { + // Claiming is a round trip, so the guard above can be passed just before shutdown starts. + // The run must not begin under that claim, and must not leave the row behind either: the + // successor would then sit out the full expiry before it could take the job over. + let answerClaim: (rows: unknown[]) => void; + const onQuery = jest.fn().mockImplementation((sql: string) => { + if (sql.includes('INSERT INTO')) return new Promise((resolve) => (answerClaim = resolve)); + return Promise.resolve([]); + }); + + const { service } = buildService({ onQuery }); + const task = jest.fn().mockResolvedValue(undefined); + const run = service.run('SomeService::job', task); + + await settle(); + await service.shutdown(); + + answerClaim([{ owner: 'worker:1' }]); + await run; + + expect(task).not.toHaveBeenCalled(); + expect(released(onQuery)).toBe(true); + }); + it('stops tracking a run once it is done, so a later shutdown has nothing to wait for', async () => { const { service } = buildService({}); @@ -306,6 +382,23 @@ describe('CronLeaseService', () => { expect(service.takeFailures().healthy).toBe(true); }); + it('reports healthy again once a RENEWAL gets through, not only a claim', async () => { + // The heartbeat reports this as a state, so every operation that reaches the table has to + // clear it. A process running long jobs renews for minutes at a time without claiming + // anything new: healing on the claim alone would leave it reporting a failure it has already + // recovered from until its next acquire. + const onQuery = jest.fn().mockRejectedValueOnce(new Error('connection refused')); + const { service } = buildService({ onQuery }); + + await service.onModuleInit(); + expect(service.takeFailures().healthy).toBe(false); + + onQuery.mockResolvedValue([[], 1]); + await service.renew('SomeService::job'); + + expect(service.takeFailures().healthy).toBe(true); + }); + it('starts out healthy, so the heartbeat does not cry wolf before anything ran', () => { const { service } = buildService({}); diff --git a/src/shared/services/cron-lease.service.ts b/src/shared/services/cron-lease.service.ts index f76a19a9a4..6f603fc41c 100644 --- a/src/shared/services/cron-lease.service.ts +++ b/src/shared/services/cron-lease.service.ts @@ -76,9 +76,19 @@ export class CronLeaseService implements OnModuleInit { /** * Whether the last lease operation reached the table. Sticky until one succeeds, so a role whose * jobs all sit out still reports the state rather than only the tick that first hit it. + * + * Every operation that reaches the table clears it, not just `acquire`: the heartbeat reports + * this as a STATE, and a process whose jobs are long-running renews for minutes at a time + * without acquiring anything. Healing on `acquire` alone would leave such a process reporting a + * failure it has already recovered from until its next claim. */ private healthy = true; + /** + * Set once shutdown has begun, so no further lease is taken. See `shutdown`. + */ + private shuttingDown = false; + /** Lease operations that failed since the role heartbeat last read them; see `takeFailures`. */ private failures = 0; private lastFailure?: string; @@ -117,6 +127,7 @@ export class CronLeaseService implements OnModuleInit { async onModuleInit(): Promise { try { await this.dataSource.query(`SELECT 1 FROM "cron_lease" LIMIT 1`); + this.recordSuccess(); } catch (e) { this.recordFailure(e); this.logger.error( @@ -145,7 +156,7 @@ export class CronLeaseService implements OnModuleInit { [job, this.owner, `${LEASE_TTL_SECONDS}`], ); - this.healthy = true; + this.recordSuccess(); return claimed.length > 0; } @@ -163,6 +174,8 @@ export class CronLeaseService implements OnModuleInit { [job, this.owner, `${LEASE_TTL_SECONDS}`], ); + this.recordSuccess(); + return affected > 0; } @@ -172,6 +185,8 @@ export class CronLeaseService implements OnModuleInit { */ async release(job: string): Promise { await this.dataSource.query(`DELETE FROM "cron_lease" WHERE "name" = $1 AND "owner" = $2`, [job, this.owner]); + + this.recordSuccess(); } /** @@ -188,6 +203,12 @@ export class CronLeaseService implements OnModuleInit { * PaymentCronService. Nothing else can tell the two apart, so the caller says which it is. */ async run(job: string, task: () => Promise, reportContention = false): Promise { + // Once shutdown has begun, starting a run is worse than skipping it: `shutdown` waits on the + // jobs it found when it started, and the process exits when that wait ends. A run started + // afterwards is not in that set, so it would be cut off mid-way — before the `finally` below + // releases its lease, and, more importantly, part-way through whatever it was doing. + if (this.shuttingDown) return; + let acquired: boolean; try { acquired = await this.acquire(job); @@ -207,24 +228,20 @@ export class CronLeaseService implements OnModuleInit { return; } - // Unref'd: a pending timer must never hold the process open on shutdown. - const renewal = setInterval(() => { - void this.renew(job) - .then((stillOurs) => { - if (!stillOurs) this.logger.error(`Lost the lease for ${job} while it was still running`); - }) - .catch((e) => { - this.recordFailure(e); - this.logger.error(`Could not extend the lease for ${job}`, e); - }); - }, RENEWAL_INTERVAL_MS); - renewal.unref(); + // Claiming the lease is a round trip, and shutdown can begin during it. Hand the claim straight + // back rather than start under it: the successor can then take the job over immediately. + if (this.shuttingDown) { + await this.release(job).catch(() => undefined); + return; + } + + const renewal = this.keepAlive(job); const run = (async () => { try { await task(); } finally { - clearInterval(renewal); + renewal.stop(); await this.release(job).catch((e) => { this.recordFailure(e); this.logger.error(`Could not release the lease for ${job}`, e); @@ -268,6 +285,11 @@ export class CronLeaseService implements OnModuleInit { * database that has stopped answering cannot turn this into a process that never exits. */ async shutdown(): Promise { + // Before the snapshot below, not after: the wait covers the jobs that were running when it was + // taken, and the process exits once it ends. A job that started meanwhile would not be waited + // for and would be cut off part-way through — see the guard at the top of `run`. + this.shuttingDown = true; + const running = [...this.inFlight.values()]; if (!running.length) return; @@ -300,6 +322,50 @@ export class CronLeaseService implements OnModuleInit { return taken; } + /** + * Keeps the claim for `job` alive while it runs, with one renewal outstanding at a time. + * + * A fixed interval fires whether or not the previous renewal has come back, and a database that + * answers slowly is exactly the situation this has to survive: the attempts pile up, each one + * occupying a pooled connection, and an older answer can land after a newer one. Re-arming only + * once the previous attempt has settled bounds that to a single outstanding statement. The price + * is that the renewals drift apart by however long the database takes to answer, which the lease + * TTL — three times the interval — is sized to absorb. + */ + private keepAlive(job: string): { stop: () => void } { + let stopped = false; + let timer: NodeJS.Timeout; + + const schedule = (): void => { + // Unref'd: a pending timer must never hold the process open on shutdown. + timer = setTimeout(async () => { + try { + const stillOurs = await this.renew(job); + if (!stillOurs) this.logger.error(`Lost the lease for ${job} while it was still running`); + } catch (e) { + this.recordFailure(e); + this.logger.error(`Could not extend the lease for ${job}`, e); + } + + if (!stopped) schedule(); + }, RENEWAL_INTERVAL_MS); + timer.unref(); + }; + + schedule(); + + return { + stop: () => { + stopped = true; + clearTimeout(timer); + }, + }; + } + + private recordSuccess(): void { + this.healthy = true; + } + private recordFailure(e: unknown): void { this.failures++; this.lastFailure = e instanceof Error ? e.message : String(e); From a39c71beadcbb3d70ba1e62dd1d57709878d5cbe Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:17 +0200 Subject: [PATCH 50/86] Put the lease state in every heartbeat instead of only in the failing one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The line carried the lease state only when something was wrong, so a reader had to count occurrences of a line that does not exist while healthy — which cannot tell a healthy process from one that stopped reporting. Both shapes now carry one of the literals 'lease ok' or 'lease unusable' at a fixed position right after the job count. Neither is a prefix of the other, and the only free text, the reason, comes last so it cannot forge a field after it. --- .../__tests__/dfx-cron.service.spec.ts | 58 ++++++++++++++----- src/shared/services/dfx-cron.service.ts | 24 ++++++-- 2 files changed, 63 insertions(+), 19 deletions(-) diff --git a/src/shared/services/__tests__/dfx-cron.service.spec.ts b/src/shared/services/__tests__/dfx-cron.service.spec.ts index 036d044831..41eab4493d 100644 --- a/src/shared/services/__tests__/dfx-cron.service.spec.ts +++ b/src/shared/services/__tests__/dfx-cron.service.spec.ts @@ -345,7 +345,7 @@ describe('DfxCronService', () => { // Three worker/both jobs plus reportRole itself. The role is what the alert matches on; the // count tells the reader on call whether the process registered a plausible number of jobs. - expect(info).toHaveBeenCalledWith('CronRole worker: heartbeat, 4 jobs registered'); + expect(info).toHaveBeenCalledWith('CronRole worker: heartbeat, 4 jobs registered, lease ok'); }); it('reports an unusable lease instead of the healthy line', () => { @@ -382,26 +382,54 @@ describe('DfxCronService', () => { // directly to it. Pinned as one expression rather than two `toContain`s: what the alert // needs is the ADJACENCY — a state reported somewhere else in the line, or in a line of its // own, would leave that alert silent while every looser assertion still passed. - expect(line).toMatch(/CronRole (api|worker|all): heartbeat, [0-9]+ jobs registered, lease unusable/); - expect(line).toContain('relation "cron_lease" does not exist'); + expect(line).toMatch(/CronRole (api|worker|all): heartbeat, [0-9]+ jobs registered, lease unusable: /); + + // The reason is free text, so it goes LAST — behind everything that is matched. Between two + // matched fields it could forge whichever one follows it. + expect(line.endsWith('relation "cron_lease" does not exist')).toBe(true); }); - it('produces a line the alert query actually matches', () => { - // The alert reads this line with `CronRole (api|worker|all): heartbeat, [0-9]+ jobs - // registered`. Pinning the wording here is the only place that couples the two: nothing in - // this repository fails if the message drifts, the alert just goes quiet. - process.env.CRON_ROLE = 'api'; + it('carries the lease state in both directions, at a fixed position', () => { + // The point of the shape: the state is in EVERY heartbeat, so a reader takes the current + // state out of one line. The previous form appended the state only when something was wrong, + // which left a reader counting occurrences of a line that does not exist while healthy — + // and a count over a window cannot tell "healthy" from "not reporting at all". + process.env.CRON_ROLE = 'worker'; new ConfigService(GetConfig()); - const { service } = buildService(configuredJobs); - service.onModuleInit(); - - const info = jest.spyOn(service['logger'], 'info'); - service.reportRole(); + const lineFor = (lease: { healthy: boolean; count: number; last?: string }): string => { + const service = new DfxCronService( + createMock({ getProviders: () => [] }), + createMock({ getAllMethodNames: () => [] }), + createMock(), + createMock({ takeFailures: () => lease }), + ); + + const info = jest.spyOn(service['logger'], 'info'); + const error = jest.spyOn(service['logger'], 'error'); + service.reportRole(); + + return (info.mock.calls[0]?.[0] ?? error.mock.calls[0]?.[0]) as string; + }; + + const healthy = lineFor({ healthy: true, count: 0 }); + const unusable = lineFor({ healthy: false, count: 1, last: 'lease ok' }); + + // Both shapes, in full. `lease ok` is not a prefix of `lease unusable`, so neither selector + // can match the other line — including when the free-text reason is itself `lease ok`, which + // is what a field order that put the reason first would fall for. + expect(healthy).toEqual('CronRole worker: heartbeat, 0 jobs registered, lease ok'); + expect(unusable).toEqual( + 'CronRole worker: heartbeat, 0 jobs registered, lease unusable: 1 failure(s) since the last heartbeat, last error: lease ok', + ); - const line = info.mock.calls[0][0] as string; + const healthySelector = /jobs registered, lease ok$/; + const unusableSelector = /jobs registered, lease unusable: /; - expect(line).toMatch(/CronRole (api|worker|all): heartbeat, [0-9]+ jobs registered/); + expect(healthySelector.test(healthy)).toBe(true); + expect(healthySelector.test(unusable)).toBe(false); + expect(unusableSelector.test(unusable)).toBe(true); + expect(unusableSelector.test(healthy)).toBe(false); }); }); }); diff --git a/src/shared/services/dfx-cron.service.ts b/src/shared/services/dfx-cron.service.ts index 2773eb40cd..d7d3063896 100644 --- a/src/shared/services/dfx-cron.service.ts +++ b/src/shared/services/dfx-cron.service.ts @@ -90,6 +90,23 @@ export class DfxCronService implements OnModuleInit { * Deliberately without a `process` flag: a watchdog that can be switched off looks, once it is * off, exactly like a process that stopped writing the line — the alert could not tell the two * apart. The job holds no state and does nothing but log, so there is nothing to switch off. + * + * The line is written to be read by a machine, in exactly one of two shapes: + * + * ``` + * CronRole : heartbeat, jobs registered, lease ok + * CronRole : heartbeat, jobs registered, lease unusable: + * ``` + * + * Three properties make that safe to match on, and all three are load-bearing. One of `lease ok` + * or `lease unusable` is ALWAYS present, so a reader sees the current state rather than having + * to count occurrences of a line that only appears when something is wrong — a count over a + * window cannot tell "healthy" from "not reporting at all". Neither literal is a prefix of the + * other, and both sit at a fixed position, immediately after the job count. And the only free + * text — the reason — comes last, behind everything that is matched, because a free-text field + * BETWEEN matched fields can forge whatever field follows it. + * + * `__tests__/dfx-cron.service.spec.ts` pins both shapes; changing the wording here fails there. */ // `useDelay: false`: the alert reads this line over a 12-minute window. With the default jitter // the gap between two heartbeats can reach 660 s, leaving 60 s of margin — and the jitter is @@ -101,13 +118,12 @@ export class DfxCronService implements OnModuleInit { const line = `CronRole ${Config.cronRole}: heartbeat, ${this.registeredCount} jobs registered`; const lease = this.leases.takeFailures(); - if (lease.healthy) return this.logger.info(line); - // A job that cannot take its lease does not run, and nothing else says so — the skip looks // exactly like a job with nothing to do. This job is scope `both` and therefore exempt from // the lease itself, so it keeps reporting while everything it counts is sitting out: a count - // of REGISTERED jobs cannot see that. Same line, because the role alert matches on its shape; - // the state is appended and the level raised. + // of REGISTERED jobs cannot see that. + if (lease.healthy) return this.logger.info(`${line}, lease ok`); + this.logger.error( `${line}, lease unusable: ${lease.count} failure(s) since the last heartbeat, last error: ${ lease.last ?? 'unknown' From 49d6f0599a12851995cc680ccdab0c2539d7a4d9 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:18 +0200 Subject: [PATCH 51/86] Drop the lease exception from the cron registration guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The lease renewal no longer uses setInterval, so the exception pointed at a file the check would have passed anyway — and would have silently granted the next setInterval added there. The reason it is invisible to the check moves into the paragraph that already lists the self-rearming timers. --- .../__tests__/cron-registration.guard.spec.ts | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/shared/services/__tests__/cron-registration.guard.spec.ts b/src/shared/services/__tests__/cron-registration.guard.spec.ts index f7ef002ca3..d0f8013d3d 100644 --- a/src/shared/services/__tests__/cron-registration.guard.spec.ts +++ b/src/shared/services/__tests__/cron-registration.guard.spec.ts @@ -17,9 +17,12 @@ const SRC = join(__dirname, '..', '..', '..'); * needs an AST-based rule rather than a text match. What this covers is the shape the four cases * in this repository actually had. * - * The setTimeout gap is not hypothetical. ScryptService.scheduleCatchUpRetry and - * ScryptWebSocketConnection.scheduleReconnect both re-arm themselves and are invisible here. They - * are deliberately left alone: their state is the process-local cache and socket of the process + * The setTimeout gap is not hypothetical. ScryptService.scheduleCatchUpRetry, + * ScryptWebSocketConnection.scheduleReconnect and CronLeaseService.keepAlive all re-arm themselves + * and are invisible here. The lease renewal belongs to the lifetime of a single job run rather + * than to a schedule, and routing it through @DfxCron would be circular — it is the mechanism that + * keeps @DfxCron jobs from running in two processes at once. The Scrypt two are deliberately left + * alone as well: their state is the process-local cache and socket of the process * they run in, and a request path reaches them (ExchangeController injects ExchangeRegistryService * and ExchangeTxService), so both processes need their own. Binding them to a role would break the * exchange endpoints on the API process. Anyone extending this check should read that case first — @@ -44,12 +47,7 @@ const FORBIDDEN: { pattern: RegExp; what: string; instead: string }[] = [ ]; /** Timers tied to the lifetime of something other than a schedule. */ -const ALLOWED = [ - // The lease renewal is bound to the lifetime of a single job run, not to a schedule: it starts - // when that run takes the lease and is cleared in its `finally`. Routing it through @DfxCron - // would be circular — it is the mechanism that keeps @DfxCron jobs from running twice. - 'shared/services/cron-lease.service.ts', -]; +const ALLOWED: string[] = []; function sourceFiles(dir: string): string[] { return readdirSync(dir).flatMap((entry) => { From 80080e49d5312fea012320601e5bb24c26c641e7 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:20 +0200 Subject: [PATCH 52/86] Give every waiter and every device entry an owner that removes it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit waitForPayment waited without an upper bound. A client that hangs up says nothing the server can hear, so its entry stayed in the wait map and in waitStates for the lifetime of the process — that is where the growth came from, not from a side effect of it. The wait is now bounded and the waiter clears its own entry on the way out, whichever way it leaves. The endpoints keep their shape: when the bound elapses they answer with the payment as it stands. The device register is no longer mirrored into the service. The gateway owns the sockets, so it is asked what is connected instead of reporting connects and disconnects into a second map that could drift from them — which is what the reference count did when one close path fired twice. Registration is bound to the socket and to every way it can end, including the error path, and a ping/pong sweep drops sockets that stopped answering, since a peer that disappears without closing fires no event at all. The delivery window is taken per device rather than as one minimum across all of them, where the quietest device set the window for everyone. --- CONTRIBUTING.md | 4 +- docs/cron-jobs.md | 26 +-- src/shared/utils/async-map.ts | 4 + .../__tests__/payment-link.gateway.spec.ts | 200 ++++++++++++++++++ .../controllers/payment-link.gateway.ts | 119 +++++++++-- .../payment-link-payment.service.spec.ts | 115 +++++++++- .../services/payment-link-payment.service.ts | 139 ++++++++---- 7 files changed, 519 insertions(+), 88 deletions(-) create mode 100644 src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e66201a9be..12d600e3aa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1047,8 +1047,8 @@ Endpoints that block by design: | Path | Blocks until | `wait` segment | | ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------- | -| `GET /v1/lnurlp/wait/:id` | the payment resolves: completed, canceled, expired — or, for `MULTIPLE`-mode links, when a quote reaches the configured completion threshold (the payment itself may stay `Pending`) | yes | -| `GET /v1/paymentLink/payment/wait` | the same, for the authenticated payment-link flow | yes | +| `GET /v1/lnurlp/wait/:id` | the payment resolves: completed, canceled, expired — or, for `MULTIPLE`-mode links, when a quote reaches the configured completion threshold (the payment itself may stay `Pending`); bounded at 60 s, after which it answers with the payment as it stands | yes | +| `GET /v1/paymentLink/payment/wait` | the same, and under the same bound, for the authenticated payment-link flow | yes | | `GET /v1/lnurlp/:id` | a pending payment appears; bounded by `timeout` (default 10 s, caller-controllable) | no — exempt | | `GET /v1/lnurlp/tx/:id` | the payer's own broadcast reaches one confirmation (`tx` branch); 15 polls at 1 s. The `hex` branch broadcasts without awaiting confirmation, except on ICP, where it first waits for the payer's allowance (up to 3 attempts, 2 s apart) | no — exempt | | `GET /v1/node/:node/tx/:txId` | the transaction reaches one confirmation; bounded at 600 s | no — exempt | diff --git a/docs/cron-jobs.md b/docs/cron-jobs.md index 032b093c16..8a37bd1b19 100644 --- a/docs/cron-jobs.md +++ b/docs/cron-jobs.md @@ -1,6 +1,6 @@ # Cron jobs -Every scheduled job this service runs: **139 `@DfxCron` declarations** across 97 files and 34 areas. +Every scheduled job this service runs: **140 `@DfxCron` declarations** across 98 files and 34 areas. ## Columns @@ -15,7 +15,7 @@ Every scheduled job this service runs: **139 `@DfxCron` declarations** across 97 ## Scopes `scope` is a mandatory parameter of `@DfxCron` and says which process registers the job: -119 are `worker`, 5 are `api`, 15 are `both`. `CRON_ROLE` decides what a process is +119 are `worker`, 5 are `api`, 16 are `both`. `CRON_ROLE` decides what a process is (`worker`, `api`, or `all` for a single-process setup); a process runs its own scope plus `both`. `worker` is the normal case — anything writing to the database or driving business forward belongs @@ -31,18 +31,19 @@ request path loads on demand, and a job may refresh it but must not be the only ## Flags -118 of the 139 jobs carry a `process` flag, 21 do not. A job with a flag can be switched off +118 of the 140 jobs carry a `process` flag, 22 do not. A job with a flag can be switched off without a deploy — `DfxCronService` skips it when the process appears in the disabled set, which `ProcessService` refreshes from the `disabledProcesses` setting and the `DISABLED_PROCESSES` environment variable every 30 seconds. -A job **without** a flag runs unconditionally. That is deliberate for five of them. The four +A job **without** a flag runs unconditionally. That is deliberate for six of them. The four `ProcessService::resync*` jobs maintain the disabled set, the JWT denylists and the staff clearance allowlist themselves, so making them switchable would let a configuration change disable the mechanism that reads configuration changes. `DfxCronService::reportRole` is the role -heartbeat the `dfx-api-role-mismatch` alert reads: switched off, it would look exactly like a -process that stopped reporting, which is the condition the alert exists to catch. For the -remaining 16 it is simply an omission: +heartbeat: switched off, it would look exactly like a process that stopped reporting, which is +the condition it exists to make visible. `PaymentLinkGateway::checkConnections` drops websockets +that stopped answering, so switching it off would reinstate the unbounded growth it prevents. For +the remaining 16 it is simply an omission: | Job | Interval | | --- | --- | @@ -64,7 +65,7 @@ New jobs should declare a flag unless there is a reason like the one above. | second | 5 | | 10 seconds | 3 | | 15 seconds | 1 | -| 30 seconds | 9 | +| 30 seconds | 10 | | minute | 52 | | 5 minutes | 18 | | 10 minutes | 16 | @@ -92,7 +93,7 @@ Jobs by area: | `shared/services` | 5 | 5 | | `subdomains/core/sell-crypto` | 5 | 2 | | `subdomains/supporting/payment` | 5 | 1 | -| `subdomains/core/payment-link` | 5 | — | +| `subdomains/core/payment-link` | 6 | 1 | | `subdomains/generic/kyc` | 4 | — | | `subdomains/supporting/bank` | 4 | — | | `subdomains/supporting/bank-tx` | 4 | — | @@ -121,9 +122,9 @@ Jobs by area: ## How this list is produced Every `@DfxCron(` occurrence in `src/**/*.ts`. Decorator arguments are read by a balanced-paren -scan, so multi-line declarations are included — a line-based match misses 26 of them. Interval, +scan, so multi-line declarations are included — a line-based match misses 27 of them. Interval, flag and scope come from those arguments, so all three are as accurate as the source. The parsed -count is asserted against a raw text count of the decorator: **139 = 139**, no gap. Class and +count is asserted against a raw text count of the decorator: **140 = 140**, no gap. Class and method come from the enclosing `export class` (including `export abstract class`) and the identifier following the decorator. @@ -147,7 +148,7 @@ the job is registered — on the provider instance, which is a different object instance the request handlers use. Resolving either one is a decision about the jobs, not about this inventory, so both are recorded -here rather than fixed in passing. Of the 139 declarations, 138 have a registration path. +here rather than fixed in passing. Of the 140 declarations, 139 have a registration path. ## Jobs @@ -163,6 +164,7 @@ here rather than fixed in passing. Of the 139 declarations, 138 have a registrat | 10 seconds | `MONITOR_EVENT_LOOP` | `both` | `MonitorEventLoopService::monitorEventLoop` | `subdomains/core/monitoring/monitor-event-loop.service.ts` | | 15 seconds | `PAYMENT_DELIVERY` | `both` | `PaymentCronService::deliverPaymentUpdates` | `subdomains/core/payment-link/services/payment-cron.service.ts` | | 30 seconds | `LNURL_AUTH_CACHE` | `both` | `AuthLnUrlService::processCleanupAccessToken` | `subdomains/generic/user/models/auth/auth-lnurl.service.ts` | +| 30 seconds | — | `both` | `PaymentLinkGateway::checkConnections` | `subdomains/core/payment-link/controllers/payment-link.gateway.ts` | | 30 seconds | `BANK_TX` | `worker` | `BankTxService::checkBankTx` | `subdomains/supporting/bank-tx/bank-tx/services/bank-tx.service.ts` | | 30 seconds | `DEX_PURCHASE_ORDER` | `worker` | `DexService::finalizePurchaseOrders` | `subdomains/supporting/dex/services/dex.service.ts` | | 30 seconds | — | `api` | `ExchangeController::checkTrades` | `integration/exchange/controllers/exchange.controller.ts` | diff --git a/src/shared/utils/async-map.ts b/src/shared/utils/async-map.ts index 445c0bdf7c..6ebea83ac3 100644 --- a/src/shared/utils/async-map.ts +++ b/src/shared/utils/async-map.ts @@ -37,6 +37,10 @@ export class AsyncMap { return Array.from(this.subscribers.keys()); } + public has(id: K): boolean { + return this.subscribers.has(id); + } + public resolve(id: K, value: T) { const subscriber = this.subscribers.get(id); if (subscriber) { diff --git a/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts b/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts new file mode 100644 index 0000000000..09e4d380e2 --- /dev/null +++ b/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts @@ -0,0 +1,200 @@ +import { IncomingMessage } from 'http'; +import { Subject } from 'rxjs'; +import { CronScope, DFX_CRONJOB_PARAMS, DfxCronParams } from 'src/shared/utils/cron'; +import { PaymentDevice } from '../../entities/payment-link-payment.entity'; +import { PaymentLinkPaymentService } from '../../services/payment-link-payment.service'; +import { PaymentLinkGateway } from '../payment-link.gateway'; + +/** + * The gateway owns the sockets, so it is the only thing that knows which devices this process can + * deliver to. Every test here is about that ownership: an entry exists for as long as a socket + * does, and for no longer — whichever way the socket ends, and whether or not it ends politely. + */ +describe('PaymentLinkGateway', () => { + let gateway: PaymentLinkGateway; + let paymentService: jest.Mocked; + let activations: Subject; + + /** A socket that records what was done to it and lets a test fire its events. */ + function socket() { + const listeners = new Map void)[]>(); + + return { + sent: [] as string[], + pings: 0, + terminated: false, + send(data: string) { + this.sent.push(data); + }, + ping() { + this.pings++; + }, + terminate() { + this.terminated = true; + }, + on(event: string, listener: () => void) { + listeners.set(event, [...(listeners.get(event) ?? []), listener]); + }, + fire(event: string) { + for (const listener of listeners.get(event) ?? []) listener(); + }, + }; + } + + /** Accepts a connection the way the adapter does, through the public entry point. */ + function connect(device: string): ReturnType { + const client = socket(); + gateway.handleConnection(client, { url: `/v1/paymentLink?device=${device}` } as IncomingMessage); + + return client; + } + + const deviceIds = () => gateway.connectedDevices().map((d) => d.id); + + beforeEach(() => { + activations = new Subject(); + paymentService = { + getDeviceActivationObservable: () => activations.asObservable(), + useDeviceSource: jest.fn(), + } as unknown as jest.Mocked; + + gateway = new PaymentLinkGateway(paymentService); + }); + + it('rejects a connection that names no device', () => { + expect(() => gateway.handleConnection(socket(), { url: '/v1/paymentLink' } as IncomingMessage)).toThrow( + 'device should not be empty', + ); + }); + + describe('what the delivery is allowed to see', () => { + it('hands the delivery a live view rather than a snapshot', () => { + // The whole point of deriving instead of mirroring: one function, asked twice, gives two + // different answers because the sockets changed underneath it. A register the gateway + // reported into could only be as current as its last report. + gateway.onModuleInit(); + + const source = (paymentService.useDeviceSource as jest.Mock).mock.calls[0][0] as () => { id: string }[]; + + expect(source()).toEqual([]); + + const client = connect('pos-1'); + expect(source().map((d) => d.id)).toEqual(['pos-1']); + + client.fire('close'); + expect(source()).toEqual([]); + }); + + it('stops reporting a device once its last connection is gone', () => { + const client = connect('pos-1'); + expect(deviceIds()).toEqual(['pos-1']); + + client.fire('close'); + + expect(deviceIds()).toEqual([]); + }); + + it('keeps reporting a device whose other connection is still open', () => { + // The failure a reference count had: one close path taken twice pushed the count below the + // number of live connections, and the device stopped being delivered to while someone was + // still listening. There is no count to push. + const first = connect('pos-1'); + const second = connect('pos-1'); + + first.fire('close'); + first.fire('close'); + + expect(deviceIds()).toEqual(['pos-1']); + + second.fire('close'); + + expect(deviceIds()).toEqual([]); + }); + + it('drops a connection that ends with an error instead of a close', () => { + // An aborted connection does not necessarily reach the close path, which is how an entry + // came to outlive its socket in the first place. + const client = connect('pos-1'); + + client.fire('error'); + + expect(deviceIds()).toEqual([]); + }); + + it('dates a device by its oldest open connection', () => { + // That date is where the delivery starts reading for a device it has not sent anything to + // yet, so it has to cover every connection, not just the newest. + jest.useFakeTimers({ doNotFake: ['setImmediate', 'nextTick'] }); + + try { + jest.setSystemTime(new Date('2026-01-01T10:00:00Z')); + const first = connect('pos-1'); + + jest.setSystemTime(new Date('2026-01-01T11:00:00Z')); + connect('pos-1'); + + expect(gateway.connectedDevices()[0].since).toEqual(new Date('2026-01-01T10:00:00Z')); + + first.fire('close'); + + expect(gateway.connectedDevices()[0].since).toEqual(new Date('2026-01-01T11:00:00Z')); + } finally { + jest.useRealTimers(); + } + }); + }); + + describe('sockets that stopped answering', () => { + it('drops one that misses the ping', () => { + // Nothing else can see this happen: a peer that vanishes without closing leaves the socket + // open here and fires no event at all, so an unanswered ping is the only evidence there is. + const client = connect('pos-1'); + + gateway.checkConnections(); + + expect(client.pings).toEqual(1); + expect(deviceIds()).toEqual(['pos-1']); + + gateway.checkConnections(); + + expect(client.terminated).toBe(true); + expect(deviceIds()).toEqual([]); + }); + + it('keeps one that answers the ping', () => { + // The negative side of the same check. Without it the sweep could pass by dropping every + // connection it looked at. + const client = connect('pos-1'); + + gateway.checkConnections(); + client.fire('pong'); + gateway.checkConnections(); + + expect(client.terminated).toBe(false); + expect(deviceIds()).toEqual(['pos-1']); + }); + + it('runs in every process, because every process holds its own sockets', () => { + const params: DfxCronParams = Reflect.getMetadata( + DFX_CRONJOB_PARAMS, + PaymentLinkGateway.prototype.checkConnections, + ); + + expect(params.scope).toEqual(CronScope.BOTH); + }); + }); + + it('sends a command to every connection of the addressed device', () => { + gateway.onModuleInit(); + + const first = connect('pos-1'); + const second = connect('pos-1'); + const other = connect('pos-2'); + + activations.next({ id: 'pos-1', command: 'show-paid' }); + + expect(first.sent).toEqual(['show-paid']); + expect(second.sent).toEqual(['show-paid']); + expect(other.sent).toEqual([]); + }); +}); diff --git a/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts b/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts index 2dcdd88461..7fd4c0ae7d 100644 --- a/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts +++ b/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts @@ -1,58 +1,135 @@ import { BadRequestException, OnModuleInit } from '@nestjs/common'; +import { CronExpression } from '@nestjs/schedule'; import { OnGatewayConnection, WebSocketGateway } from '@nestjs/websockets'; import { IncomingMessage } from 'http'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Util } from 'src/shared/utils/util'; import { PaymentDevice } from '../entities/payment-link-payment.entity'; -import { PaymentLinkPaymentService } from '../services/payment-link-payment.service'; +import { ConnectedDevice, PaymentLinkPaymentService } from '../services/payment-link-payment.service'; -type ClientMap = Map>; +/** + * The part of an accepted websocket this gateway uses. + * + * Named rather than taken wholesale so it is visible what the gateway needs from the socket, and + * so its behaviour can be exercised without standing up a server. + */ +interface PaymentSocket { + send(data: string): void; + ping(): void; + terminate(): void; + on(event: 'close' | 'error' | 'pong', listener: () => void): void; +} + +/** One open websocket, and what is known about it here. */ +interface Connection { + socket: PaymentSocket; + /** When this connection was accepted. */ + since: Date; + /** Cleared before each ping and set again by the pong; one missed round means it is gone. */ + responsive: boolean; +} @WebSocketGateway({ path: '/v1/paymentLink' }) export class PaymentLinkGateway implements OnGatewayConnection, OnModuleInit { - private readonly clients: ClientMap = new Map(); + private readonly clients = new Map>(); constructor(private readonly paymentService: PaymentLinkPaymentService) {} onModuleInit() { this.paymentService.getDeviceActivationObservable().subscribe((a) => this.sendMessage(a)); + + // The delivery reads what is connected out of the map below instead of being told about it, so + // there is no second register to keep in step. See PaymentLinkPaymentService.connectedDevices. + this.paymentService.useDeviceSource(() => this.connectedDevices()); } - handleConnection(client: WebSocket, message: IncomingMessage) { + handleConnection(client: PaymentSocket, message: IncomingMessage) { const device = new URLSearchParams(message.url?.split('?')[1]).get('device'); if (!device) throw new BadRequestException('device should not be empty'); this.addClient(device, client); } + /** + * The devices this process can deliver to right now, derived from the sockets it holds open. + * + * A device appears here for exactly as long as at least one of its connections is in the map, + * and its `since` is the age of the oldest of those — the point from which a newly connected + * device has not been sent anything yet. + */ + connectedDevices(): ConnectedDevice[] { + return [...this.clients].map(([id, connections]) => ({ + id, + since: Util.minObj([...connections.values()], 'since').since, + })); + } + + /** + * Drops the connections that stopped answering. + * + * A peer that disappears without closing its socket — a network that went away, a device that + * went to sleep — leaves the socket open on this side, and no close or error event ever arrives. + * Nothing else in this class can tell such a socket from an idle one, so without this round trip + * it would stay in the map for as long as the process lives, and the delivery would keep + * querying for a device that is gone. + * + * Deliberately without a `process` flag: switching it off would reinstate exactly the unbounded + * growth it exists to prevent. It holds no state of its own and does nothing but drop sockets + * that failed to answer, so there is nothing a kill switch would usefully stop. + */ + @DfxCron(CronExpression.EVERY_30_SECONDS, { scope: CronScope.BOTH }) + checkConnections(): void { + for (const [device, connections] of this.clients) { + for (const [clientId, connection] of connections) { + if (!connection.responsive) { + connection.socket.terminate(); + this.removeClient(device, clientId); + continue; + } + + connection.responsive = false; + connection.socket.ping(); + } + } + } + // --- HELPER METHODS --- // - private addClient(device: string, client: WebSocket) { + private addClient(device: string, client: PaymentSocket) { const clientId = Util.createUniqueId('client'); - const clients = this.clients.get(device) ?? new Map(); - clients.set(clientId, client); - this.clients.set(device, clients); - - // Tells the service which devices this process can still deliver to, so its catch-up job knows - // what to look up and can stay away from the database when nothing is connected here. - this.paymentService.registerDevice(device); + const connections = this.clients.get(device) ?? new Map(); + connections.set(clientId, { socket: client, since: new Date(), responsive: true }); + this.clients.set(device, connections); - client.onclose = () => this.removeClient(device, clientId); + // Bound to the socket, and to every way it can end: an aborted connection reports `error`, and + // binding to `close` alone left it registered. Removal is idempotent, so both firing is fine. + client.on('close', () => this.removeClient(device, clientId)); + client.on('error', () => this.removeClient(device, clientId)); + client.on('pong', () => this.markResponsive(device, clientId)); } private removeClient(device: string, clientId: string) { - const clients = this.clients.get(device); - clients?.delete(clientId); - this.clients.set(device, clients); + const connections = this.clients.get(device); + if (!connections) return; + + connections.delete(clientId); + + // The device goes with its last connection. An empty map left behind would keep the device in + // `connectedDevices` above, which is the one thing that must not outlive the sockets. + if (!connections.size) this.clients.delete(device); + } - this.paymentService.unregisterDevice(device); + private markResponsive(device: string, clientId: string) { + const connection = this.clients.get(device)?.get(clientId); + if (connection) connection.responsive = true; } private sendMessage(device: PaymentDevice) { - const clients = this.clients.get(device.id); - if (!clients) return; + const connections = this.clients.get(device.id); + if (!connections) return; - for (const client of clients.values()) { - client.send(device.command); + for (const { socket } of connections.values()) { + socket.send(device.command); } } } diff --git a/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts b/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts index 6472975ad2..fb4a45ab3a 100644 --- a/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts +++ b/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts @@ -48,6 +48,17 @@ describe('PaymentLinkPaymentService', () => { return seen; } + /** + * Stands in for the gateway's socket map. The service reads the connected devices out of it on + * every delivery, so a device is connected here for exactly as long as this map says so — there + * is no register in the service to register with. + */ + let sockets: Map; + + function connect(deviceId: string, since = new Date('2026-01-01T09:00:00Z')): void { + sockets.set(deviceId, since); + } + beforeEach(() => { paymentLinkPaymentRepo = { find: jest.fn().mockResolvedValue([]), @@ -68,6 +79,9 @@ describe('PaymentLinkPaymentService', () => { paymentActivationService, {} as unknown as jest.Mocked, ); + + sockets = new Map(); + service.useDeviceSource(() => [...sockets].map(([id, since]) => ({ id, since }))); }); // --- deliverPaymentUpdates() Tests --- // @@ -122,7 +136,7 @@ describe('PaymentLinkPaymentService', () => { it('should send the command to a device connected here after another process wrote the payment', async () => { const seen = devices(); - service.registerDevice('pos-1'); + connect('pos-1'); paymentLinkPaymentRepo.find.mockResolvedValue([ payment({ @@ -140,7 +154,7 @@ describe('PaymentLinkPaymentService', () => { it('should send the same payment state to a device once', async () => { const seen = devices(); - service.registerDevice('pos-1'); + connect('pos-1'); paymentLinkPaymentRepo.find.mockResolvedValue([ payment({ @@ -157,19 +171,102 @@ describe('PaymentLinkPaymentService', () => { expect(seen).toHaveLength(1); }); - it('should stop looking for a device whose last connection closed', async () => { - service.registerDevice('pos-1'); - service.registerDevice('pos-1'); - service.unregisterDevice('pos-1'); + it('should stop looking for a device the moment the gateway no longer holds it', async () => { + // Nothing tells the service the device went away, and nothing has to: it reads the connected + // devices on every delivery, so a device that is gone simply stops appearing. + connect('pos-1'); await service.deliverPaymentUpdates(); expect(paymentLinkPaymentRepo.find).toHaveBeenCalledTimes(1); - service.unregisterDevice('pos-1'); + sockets.delete('pos-1'); await service.deliverPaymentUpdates(); expect(paymentLinkPaymentRepo.find).toHaveBeenCalledTimes(1); }); + + it('should ask each device for its own window, not for the oldest one of all', async () => { + // A single minimum across all devices lets the quietest one set the window for everyone: the + // busy device is then re-read from the point the quiet one connected, on every tick. + const seen = devices(); + connect('pos-1', new Date('2026-01-01T09:00:00Z')); + + paymentLinkPaymentRepo.find.mockResolvedValue([ + payment({ + id: 7, + status: PaymentLinkPaymentStatus.COMPLETED, + deviceId: 'pos-1', + deviceCommand: 'show-paid', + updated: new Date('2026-01-01T11:00:00Z'), + }), + ]); + await service.deliverPaymentUpdates(); + expect(seen).toHaveLength(1); + + // A second device joins, connected long before anything happened on the first one. + connect('pos-2', new Date('2026-01-01T08:00:00Z')); + paymentLinkPaymentRepo.find.mockClear(); + await service.deliverPaymentUpdates(); + + const { where } = paymentLinkPaymentRepo.find.mock.calls[0][0]; + const windows = new Map((where as { deviceId: string; updated: { value: Date } }[]).map((w) => [w.deviceId, w])); + + // The device that was already delivered to has moved on; the newcomer starts at its own + // connection time. One shared window could not express both. + expect(windows.get('pos-1').updated.value).toEqual(new Date('2026-01-01T11:00:00Z')); + expect(windows.get('pos-2').updated.value).toEqual(new Date('2026-01-01T08:00:00Z')); + }); + }); + + // --- waitForPayment() Tests --- // + + describe('waitForPayment()', () => { + it('should answer with the payment on hand once the wait elapses', async () => { + // The endpoints keep their shape: a caller that is still there is told what the payment looks + // like now, which for a pending one means "not yet, ask again". + jest.useFakeTimers({ doNotFake: ['setImmediate', 'nextTick'] }); + + try { + const pending = payment({ id: 7 }); + const waiting = service.waitForPayment(pending); + + jest.advanceTimersByTime(60_000); + + await expect(waiting).resolves.toBe(pending); + } finally { + jest.useRealTimers(); + } + }); + + it('should leave nothing behind when the wait elapses', async () => { + // The reason the wait is bounded at all. A client that hangs up says nothing the server can + // hear, so an unbounded wait left both entries in place for the lifetime of the process. + jest.useFakeTimers({ doNotFake: ['setImmediate', 'nextTick'] }); + + try { + const waiting = service.waitForPayment(payment({ id: 7 })); + + jest.advanceTimersByTime(60_000); + await waiting; + + expect(service['waitStates'].size).toEqual(0); + expect(service['paymentWaitMap'].get()).toEqual([]); + } finally { + jest.useRealTimers(); + } + }); + + it('should keep the compared state while a wait on the same payment is still registered', async () => { + // Two callers share one entry, so the one leaving must not take the state the other is + // comparing against with it. + const first = service.waitForPayment(payment({ id: 7 })); + void service.waitForPayment(payment({ id: 7 })); + + paymentLinkPaymentRepo.find.mockResolvedValue([payment({ id: 7, status: PaymentLinkPaymentStatus.COMPLETED })]); + await service.deliverPaymentUpdates(); + + await expect(first).resolves.toMatchObject({ status: PaymentLinkPaymentStatus.COMPLETED }); + }); }); // --- doSave() Tests --- // @@ -178,7 +275,7 @@ describe('PaymentLinkPaymentService', () => { it('should release a caller in the writing process without waiting for the job', async () => { const pending = payment({ id: 7, deviceId: 'pos-1', deviceCommand: 'show-paid' }); const seen = devices(); - service.registerDevice('pos-1'); + connect('pos-1'); const waiting = service.waitForPayment(pending); await service.expirePayment(pending); @@ -190,7 +287,7 @@ describe('PaymentLinkPaymentService', () => { it('should not repeat a delivery the writing process already made', async () => { const pending = payment({ id: 7, deviceId: 'pos-1', deviceCommand: 'show-paid' }); const seen = devices(); - service.registerDevice('pos-1'); + connect('pos-1'); await service.expirePayment(pending); diff --git a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts index 221c88ad07..d3bfdf3922 100644 --- a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts @@ -33,11 +33,29 @@ import { PaymentActivationService } from './payment-activation.service'; import { PaymentQuoteService } from './payment-quote.service'; import { PaymentWebhookService } from './payment-webhook.service'; -/** What one process knows about the websocket connections it holds open for a single device. */ -interface DeviceSubscription { - /** Open connections for this device on this process. The entry is dropped when the last closes. */ - connections: number; - /** Payments updated at or before this are not delivered: they predate the oldest connection. */ +/** + * How long a caller of `waitForPayment` is held before it is answered with the state on hand. + * + * A server-side long poll without an upper bound leaks by construction: a client that hangs up + * says nothing the server can hear, so its entry would sit in the maps below until the process + * restarts. The bound is what gives the entry an owner — with it, the waiter itself clears the + * entry on the way out, whichever way it leaves. + * + * The endpoints answering from this do not change shape when it elapses; they answer with the + * payment as it stands, which for a caller that is still there means "not yet, ask again". + */ +const PAYMENT_WAIT_TIMEOUT_SECONDS = 60; + +/** A device this process holds at least one open websocket connection for. */ +export interface ConnectedDevice { + id: string; + /** When the oldest connection still open for this device was accepted. */ + since: Date; +} + +/** How far this process has got in delivering to one device. Never a record of who is connected. */ +interface DeliveryCursor { + /** Payments updated at or before this have been delivered, or predate the oldest connection. */ since: Date; /** `:` of the last command delivered, so the same state is sent once. */ delivered?: string; @@ -48,7 +66,23 @@ export class PaymentLinkPaymentService { private readonly paymentWaitMap = new AsyncMap(this.constructor.name); private readonly waitStates = new Map(); private readonly deviceActivationSubject = new Subject(); - private readonly deviceSubscriptions = new Map(); + private readonly deviceCursors = new Map(); + + /** + * Where the connected devices are READ from — set once by PaymentLinkGateway, which owns the + * sockets. + * + * A device is connected for exactly as long as the gateway holds a socket for it, so the socket + * map is the only thing that knows. Reading through to it beats having the gateway report + * connects and disconnects into a second map here: a mirrored register can drift from what it + * mirrors, and every way it drifted was a defect — an entry left behind when no close event + * arrived, a count decremented twice by a close path taken twice, an entry with no counterpart. + * A derived register has no second copy to get out of step with. + * + * Empty until the gateway sets it, which is also the honest answer for a process that accepts no + * websocket connections at all. + */ + private connectedDevices: () => ConnectedDevice[] = () => []; constructor( private readonly fiatService: FiatService, @@ -193,27 +227,26 @@ export class PaymentLinkPaymentService { // change from it, not a fixed target state (see PaymentLinkPayment.waitState). if (!this.waitStates.has(payment.id)) this.waitStates.set(payment.id, payment.waitState); - return this.paymentWaitMap.wait(payment.id, 0); - } - - /** Called by PaymentLinkGateway for every websocket connection it accepts. */ - registerDevice(deviceId: string): void { - const subscription = this.deviceSubscriptions.get(deviceId); - - if (subscription) { - subscription.connections++; - } else { - this.deviceSubscriptions.set(deviceId, { connections: 1, since: new Date() }); + try { + return await this.paymentWaitMap.wait(payment.id, PAYMENT_WAIT_TIMEOUT_SECONDS * 1000); + } catch { + // The wait elapsed. Answer with the payment as it stands rather than fail: the caller asked + // whether anything happened, and "not within this window" is an answer to that. + return payment; + } finally { + // The waiter owns its entry. `resolveWaiters` clears it on the delivery path; this clears it + // on every other one — which is the path that used to have no owner at all. Guarded on the + // wait map so a wait registered in the meantime keeps the state it is comparing against. + if (!this.paymentWaitMap.has(payment.id)) this.waitStates.delete(payment.id); } } - /** Called by PaymentLinkGateway when one of those connections closes. */ - unregisterDevice(deviceId: string): void { - const subscription = this.deviceSubscriptions.get(deviceId); - if (!subscription) return; - - subscription.connections--; - if (subscription.connections < 1) this.deviceSubscriptions.delete(deviceId); + /** + * Points the delivery below at the gateway's socket map; see `connectedDevices`. Called once, by + * PaymentLinkGateway, which is the only thing that knows what is connected here. + */ + useDeviceSource(source: () => ConnectedDevice[]): void { + this.connectedDevices = source; } /** @@ -245,28 +278,44 @@ export class PaymentLinkPaymentService { } private async deliverToConnectedDevices(): Promise { - const subscriptions = Array.from(this.deviceSubscriptions.entries()); - if (!subscriptions.length) return; + const devices = this.connectedDevices(); - const deviceIds = subscriptions.map(([deviceId]) => deviceId); - const since = Util.minObj( - subscriptions.map(([, subscription]) => subscription), - 'since', - ).since; + // A cursor is a delivery detail of this process, so it follows the connections rather than + // outliving them. Pruning here rather than on a disconnect notification is the point: nothing + // has to be told that a device went away, it simply stops appearing. + for (const deviceId of this.deviceCursors.keys()) { + if (!devices.some((device) => device.id === deviceId)) this.deviceCursors.delete(deviceId); + } - // The same condition the direct delivery in doSave runs under, expressed over stored columns: - // a payment out of `Pending`, or a `MULTIPLE`-mode payment that has counted a completed quote. - const payments = await this.paymentLinkPaymentRepo.find({ - where: [ - { deviceId: In(deviceIds), updated: MoreThan(since), status: Not(PaymentLinkPaymentStatus.PENDING) }, - { deviceId: In(deviceIds), updated: MoreThan(since), txCount: MoreThan(0) }, - ], - order: { updated: 'ASC' }, + if (!devices.length) return; + + // One window PER DEVICE. Taken as a single minimum across all of them, the device connected + // longest — or simply the quietest — sets the window for every other one, and every tick then + // re-reads what those have already been sent. The condition inside each window is the one the + // direct delivery in doSave runs under, expressed over stored columns: a payment out of + // `Pending`, or a `MULTIPLE`-mode payment that has counted a completed quote. + const where = devices.flatMap((device) => { + const { since } = this.cursorFor(device); + + return [ + { deviceId: device.id, updated: MoreThan(since), status: Not(PaymentLinkPaymentStatus.PENDING) }, + { deviceId: device.id, updated: MoreThan(since), txCount: MoreThan(0) }, + ]; }); + const payments = await this.paymentLinkPaymentRepo.find({ where, order: { updated: 'ASC' } }); + for (const payment of payments) this.deliverToDevice(payment); } + /** The cursor for a connected device, starting at the age of its oldest open connection. */ + private cursorFor(device: ConnectedDevice): DeliveryCursor { + const cursor = this.deviceCursors.get(device.id) ?? { since: device.since }; + this.deviceCursors.set(device.id, cursor); + + return cursor; + } + private resolveWaiters(payment: PaymentLinkPayment): void { this.waitStates.delete(payment.id); this.paymentWaitMap.resolve(payment.id, payment); @@ -280,15 +329,17 @@ export class PaymentLinkPaymentService { const device = payment.device; if (!device) return; - const subscription = this.deviceSubscriptions.get(device.id); - if (!subscription) return; + const connected = this.connectedDevices().find((d) => d.id === device.id); + if (!connected) return; + + const cursor = this.cursorFor(connected); const state = `${payment.id}:${payment.waitState}`; - if (state === subscription.delivered) return; + if (state === cursor.delivered) return; - subscription.delivered = state; + cursor.delivered = state; // Keeps the window of the query above from growing over the lifetime of a connection. - if (payment.updated > subscription.since) subscription.since = payment.updated; + if (payment.updated > cursor.since) cursor.since = payment.updated; this.deviceActivationSubject.next(device); } From d5f9c5979b6b0a88398b115d265a69662b1843ba Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:22 +0200 Subject: [PATCH 53/86] Say that the lease bounds a double run instead of excluding one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CronLeaseService already stated under 'What it does not do' that a lease can expire under a running job and let a second process start it. Twelve other places said the opposite — 'at most one process', 'closes that by construction', 'makes it structural', 'the double run the lease exists to prevent'. On a money path that difference decides what an operator does during an incident: told the double run is impossible, running a second worker looks safe. They now all say the same thing. The window is bounded by the lease TTL instead of running until someone reads an alert; what a job scoped worker rests on is a deployment that runs one worker and a job that tolerates a repeat; the lease is defence in depth over those two rather than a replacement for either. --- CONTRIBUTING.md | 14 +- migration/1785600000000-AddCronLease.js | 7 +- .../__tests__/cron-lease.service.spec.ts | 10 +- .../__tests__/cron-registration.guard.spec.ts | 2 +- .../__tests__/dfx-cron.service.spec.ts | 4 +- src/shared/services/cron-lease.service.ts | 10 +- src/shared/services/dfx-cron.service.ts | 496 +++---- .../services/payment-link-payment.service.ts | 1296 ++++++++--------- 8 files changed, 930 insertions(+), 909 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 12d600e3aa..52b5e13e05 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -539,9 +539,17 @@ looks, once it is off, exactly like the failure it watches for. Without a flag t unconditionally and cannot be switched off without a deploy. A job scoped `worker` or `api` additionally holds a **lease in the database** for the duration of -its run (`CronLeaseService`), so it runs in at most one process even if the configuration is -wrong — a missed recreate, a second worker from `--scale`, two processes on `all` after a -rollback. The in-process lock cannot see any of those. Jobs scoped `both` are exempt by design: +its run (`CronLeaseService`). The in-process lock cannot see a second process at all — a missed +recreate, a second worker from `--scale`, two processes on `all` after a rollback — and the lease +bounds how long such a configuration can have the job running twice to the lease expiry, instead +of until someone reads an alert. + +It does **not** make a double run impossible, and nothing in this repository should claim it does. +If the database becomes unreachable mid-run the claim lapses while the job keeps working, and a +second process can then start it. What a `worker` job actually rests on is the deployment running +one worker and the job tolerating a repeat; the lease is defence in depth over those two, not a +substitute for either. `CronLeaseService` states the limit under "What it does not do" — keep any +wording here consistent with it. Jobs scoped `both` are exempt by design: they must run everywhere, which is why running them twice has to be harmless by construction. [docs/cron-jobs.md](docs/cron-jobs.md) lists every scheduled job with its interval, flag and scope. diff --git a/migration/1785600000000-AddCronLease.js b/migration/1785600000000-AddCronLease.js index 0258ff17e0..cd09f6c538 100644 --- a/migration/1785600000000-AddCronLease.js +++ b/migration/1785600000000-AddCronLease.js @@ -13,8 +13,11 @@ * *reports* a double run about fifteen minutes after it starts. For a path that moves money, * detection is the second-best answer. * - * This table makes it structural: a job scoped to exactly one process must hold a row here for the - * duration of its run, and the row is claimable by only one process at a time. + * This table bounds it: a job scoped to exactly one process must hold a row here for the duration + * of its run, and the row is claimable by one process at a time until it expires. The expiry is + * what makes this a bound rather than an exclusion — if the holder can no longer renew, a second + * process can claim the row while the first is still working. See CronLeaseService, "What it does + * not do". * * No foreign keys, deliberately: the table is infrastructure, not domain data, and a key into a * domain table would tie a coordination row to a schema it has no business depending on. diff --git a/src/shared/services/__tests__/cron-lease.service.spec.ts b/src/shared/services/__tests__/cron-lease.service.spec.ts index f39c02a22d..879e0af2e7 100644 --- a/src/shared/services/__tests__/cron-lease.service.spec.ts +++ b/src/shared/services/__tests__/cron-lease.service.spec.ts @@ -6,9 +6,11 @@ import { DataSource } from 'typeorm'; import { CronLeaseService } from '../cron-lease.service'; /** - * The lease is the only thing that stops a payout from running twice when two processes disagree - * about who owns a job. Every test here is written from that angle: not "does the method return - * true", but "can this state let the task run twice, or stop it from running at all". + * The lease bounds how long two processes that disagree about who owns a job can both run it. It + * does not rule that out — the jobs' own tolerance of a repeat and a deployment that runs one + * worker are what carry that, and this is a layer over them. Every test here is written from that + * angle: not "does the method return true", but "can this state widen that window, or stop the + * task from running at all". */ describe('CronLeaseService', () => { const original = process.env.CRON_ROLE; @@ -204,7 +206,7 @@ describe('CronLeaseService', () => { it('does not take the lease away from a job that is still running', async () => { // The dangerous direction. Releasing on SIGTERM would let the successor claim the lease and // start the same job while this process keeps working on it for the rest of its stop grace - // period — the double run the lease exists to prevent. + // period — the double run the lease exists to keep short. jest.useFakeTimers({ doNotFake: ['setImmediate', 'nextTick'] }); try { diff --git a/src/shared/services/__tests__/cron-registration.guard.spec.ts b/src/shared/services/__tests__/cron-registration.guard.spec.ts index d0f8013d3d..c669cd4359 100644 --- a/src/shared/services/__tests__/cron-registration.guard.spec.ts +++ b/src/shared/services/__tests__/cron-registration.guard.spec.ts @@ -21,7 +21,7 @@ const SRC = join(__dirname, '..', '..', '..'); * ScryptWebSocketConnection.scheduleReconnect and CronLeaseService.keepAlive all re-arm themselves * and are invisible here. The lease renewal belongs to the lifetime of a single job run rather * than to a schedule, and routing it through @DfxCron would be circular — it is the mechanism that - * keeps @DfxCron jobs from running in two processes at once. The Scrypt two are deliberately left + * bounds how long a @DfxCron job can be running in two processes at once. The Scrypt two are deliberately left * alone as well: their state is the process-local cache and socket of the process * they run in, and a request path reaches them (ExchangeController injects ExchangeRegistryService * and ExchangeTxService), so both processes need their own. Binding them to a role would break the diff --git a/src/shared/services/__tests__/dfx-cron.service.spec.ts b/src/shared/services/__tests__/dfx-cron.service.spec.ts index 41eab4493d..eee011382c 100644 --- a/src/shared/services/__tests__/dfx-cron.service.spec.ts +++ b/src/shared/services/__tests__/dfx-cron.service.spec.ts @@ -156,8 +156,8 @@ describe('DfxCronService', () => { describe('cross-process lease', () => { // Which process runs a job is decided by configuration, and configuration can be wrong. The - // lease is what makes a wrong configuration harmless instead of expensive — these two tests - // pin who goes through it, because nothing at the call site shows it. + // lease bounds what a wrong configuration costs, rather than making it harmless — these two + // tests pin who goes through it, because nothing at the call site shows it. /** Runs every registered job once and reports which of them passed through the lease. */ async function leasedJobs(role: string): Promise { diff --git a/src/shared/services/cron-lease.service.ts b/src/shared/services/cron-lease.service.ts index 6f603fc41c..e94ebd3480 100644 --- a/src/shared/services/cron-lease.service.ts +++ b/src/shared/services/cron-lease.service.ts @@ -42,7 +42,13 @@ const SHUTDOWN_GRACE_MS = 10 * 1000; * ran as one process; it cannot see a second one. Since the HTTP process and the worker are split * apart, "exactly one process runs this job" rests on configuration, a runbook sentence and an * alert that *reports* a double run after the fact. For a path that moves money that is the - * second-best answer, so this makes it structural. + * second-best answer, so this adds a bound underneath it. + * + * A bound, not a guarantee — read "What it does not do" below before relying on this. A job runs + * once because the deployment runs one worker and because the job tolerates being run again; what + * this contributes is to turn the window in which a wrong configuration can have it running twice + * from unbounded — until a human reads the alert — into `LEASE_TTL_SECONDS`. It sits on top of + * those two properties and replaces neither. * * The lease is claimed per job name, and only one owner can hold it. It carries an expiry rather * than a lock held on a connection: a connection-bound `pg_advisory_lock` would occupy one pooled @@ -275,7 +281,7 @@ export class CronLeaseService implements OnModuleInit { * over faster, but the job keeps running until the container's stop grace period ends it — * `dfx-api-worker` is configured to allow two minutes — and a successor claiming the freed lease * inside that window would run the same money-moving job alongside it. That is the outcome this - * whole mechanism exists to prevent, so it is not traded for a faster handover. + * mechanism exists to keep rare and short, so it is not traded for a faster handover. * * What is still running after the wait therefore keeps its lease, which lapses within * `LEASE_TTL_SECONDS` of the last renewal. The renewal timers deliberately keep going meanwhile: diff --git a/src/shared/services/dfx-cron.service.ts b/src/shared/services/dfx-cron.service.ts index d7d3063896..c53d0943e6 100644 --- a/src/shared/services/dfx-cron.service.ts +++ b/src/shared/services/dfx-cron.service.ts @@ -1,247 +1,249 @@ -import { Injectable, OnModuleInit } from '@nestjs/common'; -import { DiscoveryService, MetadataScanner } from '@nestjs/core'; -import { CronExpression, SchedulerRegistry } from '@nestjs/schedule'; -import { CronJob } from 'cron'; -import { Config, CronRole } from 'src/config/config'; -import { DisabledProcess } from 'src/shared/services/process.service'; -import { CronScope, DFX_CRONJOB_PARAMS, DfxCron, DfxCronExpression, DfxCronParams } from 'src/shared/utils/cron'; -import { LockClass } from 'src/shared/utils/lock'; -import { CronLeaseService } from './cron-lease.service'; -import { Util } from 'src/shared/utils/util'; -import { CustomCronExpression } from '../utils/custom-cron-expression'; -import { DfxLogger } from './dfx-logger'; - -interface CronJobData { - instance: object; - methodRef: any; - methodName: string; - params: DfxCronParams; -} - -@Injectable() -export class DfxCronService implements OnModuleInit { - private readonly logger = new DfxLogger(DfxCronService); - - private registeredCount = 0; - - constructor( - private readonly discovery: DiscoveryService, - private readonly metadataScanner: MetadataScanner, - private readonly schedulerRegisty: SchedulerRegistry, - private readonly leases: CronLeaseService, - ) {} - - onModuleInit() { - const registered: CronScope[] = []; - let skipped = 0; - - this.discovery - .getProviders() - .filter((wrapper) => wrapper.isDependencyTreeStatic()) - .filter(({ instance }) => instance && Object.getPrototypeOf(instance)) - .forEach(({ instance }) => { - this.metadataScanner - .getAllMethodNames(instance) - .map((methodName) => { - const methodRef = instance[methodName]; - - return { - instance, - methodRef, - methodName, - params: Reflect.getMetadata(DFX_CRONJOB_PARAMS, methodRef), - }; - }) - .filter((data) => data.params) - .forEach((data) => { - if (!this.runsInThisRole(data.params.scope)) { - skipped++; - return; - } - - registered.push(data.params.scope); - this.addCronJob(data); - }); - }); - - // Counts what this process actually registered, which is not the same as counting decorators - // in the source: a job declared on an abstract base class is registered once per concrete - // provider, and the filter above skips providers whose dependency tree is not static. Stays - // on `info` so the split is readable without changing the log level. - const total = registered.length + skipped; - const byScope = Object.values(CronScope) - .map((scope) => `${scope}: ${registered.filter((s) => s === scope).length}`) - .join(', '); - - this.registeredCount = registered.length; - - this.logger.info(`CronRole ${Config.cronRole}: registered ${registered.length} of ${total} jobs (${byScope})`); - } - - /** - * The line above says which role this process STARTED with. It cannot answer whether the two - * processes are running the right roles right now: it is written once, so an alert built on a - * counting window over it either reports nothing after the window passes, or reports permanently. - * This line answers the same question continuously, and the alert reads it. - * - * Deliberately `both`: it has to appear in EVERY process, and it carries the role, so a swapped - * assignment shows up as a wrong role rather than only as a missing line. - * - * Deliberately without a `process` flag: a watchdog that can be switched off looks, once it is - * off, exactly like a process that stopped writing the line — the alert could not tell the two - * apart. The job holds no state and does nothing but log, so there is nothing to switch off. - * - * The line is written to be read by a machine, in exactly one of two shapes: - * - * ``` - * CronRole : heartbeat, jobs registered, lease ok - * CronRole : heartbeat, jobs registered, lease unusable: - * ``` - * - * Three properties make that safe to match on, and all three are load-bearing. One of `lease ok` - * or `lease unusable` is ALWAYS present, so a reader sees the current state rather than having - * to count occurrences of a line that only appears when something is wrong — a count over a - * window cannot tell "healthy" from "not reporting at all". Neither literal is a prefix of the - * other, and both sit at a fixed position, immediately after the job count. And the only free - * text — the reason — comes last, behind everything that is matched, because a free-text field - * BETWEEN matched fields can forge whatever field follows it. - * - * `__tests__/dfx-cron.service.spec.ts` pins both shapes; changing the wording here fails there. - */ - // `useDelay: false`: the alert reads this line over a 12-minute window. With the default jitter - // the gap between two heartbeats can reach 660 s, leaving 60 s of margin — and the jitter is - // configurable through CRON_JOB_DELAY, so someone could close that margin from the outside - // without ever seeing this code. A watchdog must not have its own timing tuned by a knob meant - // for spreading load. - @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.BOTH, useDelay: false }) - reportRole(): void { - const line = `CronRole ${Config.cronRole}: heartbeat, ${this.registeredCount} jobs registered`; - const lease = this.leases.takeFailures(); - - // A job that cannot take its lease does not run, and nothing else says so — the skip looks - // exactly like a job with nothing to do. This job is scope `both` and therefore exempt from - // the lease itself, so it keeps reporting while everything it counts is sitting out: a count - // of REGISTERED jobs cannot see that. - if (lease.healthy) return this.logger.info(`${line}, lease ok`); - - this.logger.error( - `${line}, lease unusable: ${lease.count} failure(s) since the last heartbeat, last error: ${ - lease.last ?? 'unknown' - }`, - ); - } - - /** - * `all` runs everything, which is the single-process mode. The other two roles each run their - * own scope plus `both`, so a job maintaining process-local state that requests read is - * registered in every process. - */ - private runsInThisRole(scope: CronScope): boolean { - switch (Config.cronRole) { - case CronRole.ALL: - return true; - - case CronRole.API: - return scope === CronScope.API || scope === CronScope.BOTH; - - case CronRole.WORKER: - return scope === CronScope.WORKER || scope === CronScope.BOTH; - } - } - - private addCronJob(data: CronJobData) { - const lock = LockClass.create(data.params.timeout ?? Infinity); - - const context = { target: data.instance.constructor.name, method: data.methodName }; - const cronJobName = `${context.target}::${context.method}`; - const run = this.guardAcrossProcesses(cronJobName, data); - const cronJob = new CronJob(data.params.expression, () => lock(run, context)); - - this.schedulerRegisty.addCronJob(cronJobName, cronJob); - cronJob.start(); - - this.logger.verbose(`Registered ${cronJobName} (${data.params.scope})`); - } - - /** - * Wraps a job so that at most one process in the deployment runs it at a time. - * - * `lock` above only spans this process. Which process a job belongs to is decided by - * configuration, and configuration can be wrong — a missed recreate leaves the old role in - * place, `--scale` creates a second worker, a rollback puts two processes on `all`. In every - * one of those the two processes hold separate locks and every payout runs twice. The lease - * closes that by construction instead of reporting it a quarter of an hour later. - * - * `BOTH` jobs are deliberately exempt. They exist because a request path on THIS process reads - * the state they maintain, so they have to run in every process — a lease over them would - * starve whichever process lost the race, and the job would silently stop maintaining that - * state. Their safety comes from a different property: running twice must be harmless by - * construction, which is what CONTRIBUTING requires of them. - * - * `API` jobs are leased as well, and reported when they lose the race. Their scope says their - * effect is confined to the process running them, which is an argument for every API process - * running them; the lease is what keeps one that also writes or calls out from doing either - * twice. Where the two pull apart the lease wins, and the process that lost the race is left - * without whatever the job maintains — which is why an `API` job must not be the only thing - * filling what a request path reads, and why delivering to connections belongs to a `BOTH` job - * driven from stored state rather than to this scope. Losing the race is reported rather than - * passed over, because for these jobs it is the symptom of a deployment running more than one - * API process rather than a normal cycle. - */ - private guardAcrossProcesses(cronJobName: string, data: CronJobData): () => Promise { - const task = this.wrapFunction(data); - - if (data.params.scope === CronScope.BOTH) return task; - - return () => this.leases.run(cronJobName, task, data.params.scope === CronScope.API); - } - - private wrapFunction(data: CronJobData) { - const context = { target: data.instance.constructor.name, method: data.methodName }; - - return async (...args: any) => { - if (data.params.process && DisabledProcess(data.params.process)) { - this.logger.verbose( - `Skipping ${context.target}::${context.method} - process ${data.params.process} is disabled`, - ); - return; - } - - if (data.params.useDelay ?? true) await this.cronJobDelay(data.params.expression); - - await data.methodRef.apply(data.instance, args); - }; - } - - private async cronJobDelay(expression: DfxCronExpression): Promise { - const random = Math.random() * 1000; - - const delays = Config.cronJobDelay; - - switch (expression) { - case CronExpression.EVERY_10_SECONDS: - return Util.delay(random * (delays[0] ?? 5)); - - case CustomCronExpression.EVERY_15_SECONDS: - return Util.delay(random * (delays[1] ?? 5)); - - case CronExpression.EVERY_30_SECONDS: - return Util.delay(random * (delays[2] ?? 15)); - - case CronExpression.EVERY_MINUTE: - return Util.delay(random * (delays[3] ?? 30)); - - case CronExpression.EVERY_5_MINUTES: - return Util.delay(random * (delays[4] ?? 60)); - - case CronExpression.EVERY_10_MINUTES: - return Util.delay(random * (delays[5] ?? 60)); - - case CustomCronExpression.EVERY_15_MINUTES: - return Util.delay(random * (delays[6] ?? 60)); - - case CronExpression.EVERY_HOUR: - return Util.delay(random * (delays[7] ?? 120)); - } - } -} +import { Injectable, OnModuleInit } from '@nestjs/common'; +import { DiscoveryService, MetadataScanner } from '@nestjs/core'; +import { CronExpression, SchedulerRegistry } from '@nestjs/schedule'; +import { CronJob } from 'cron'; +import { Config, CronRole } from 'src/config/config'; +import { DisabledProcess } from 'src/shared/services/process.service'; +import { CronScope, DFX_CRONJOB_PARAMS, DfxCron, DfxCronExpression, DfxCronParams } from 'src/shared/utils/cron'; +import { LockClass } from 'src/shared/utils/lock'; +import { CronLeaseService } from './cron-lease.service'; +import { Util } from 'src/shared/utils/util'; +import { CustomCronExpression } from '../utils/custom-cron-expression'; +import { DfxLogger } from './dfx-logger'; + +interface CronJobData { + instance: object; + methodRef: any; + methodName: string; + params: DfxCronParams; +} + +@Injectable() +export class DfxCronService implements OnModuleInit { + private readonly logger = new DfxLogger(DfxCronService); + + private registeredCount = 0; + + constructor( + private readonly discovery: DiscoveryService, + private readonly metadataScanner: MetadataScanner, + private readonly schedulerRegisty: SchedulerRegistry, + private readonly leases: CronLeaseService, + ) {} + + onModuleInit() { + const registered: CronScope[] = []; + let skipped = 0; + + this.discovery + .getProviders() + .filter((wrapper) => wrapper.isDependencyTreeStatic()) + .filter(({ instance }) => instance && Object.getPrototypeOf(instance)) + .forEach(({ instance }) => { + this.metadataScanner + .getAllMethodNames(instance) + .map((methodName) => { + const methodRef = instance[methodName]; + + return { + instance, + methodRef, + methodName, + params: Reflect.getMetadata(DFX_CRONJOB_PARAMS, methodRef), + }; + }) + .filter((data) => data.params) + .forEach((data) => { + if (!this.runsInThisRole(data.params.scope)) { + skipped++; + return; + } + + registered.push(data.params.scope); + this.addCronJob(data); + }); + }); + + // Counts what this process actually registered, which is not the same as counting decorators + // in the source: a job declared on an abstract base class is registered once per concrete + // provider, and the filter above skips providers whose dependency tree is not static. Stays + // on `info` so the split is readable without changing the log level. + const total = registered.length + skipped; + const byScope = Object.values(CronScope) + .map((scope) => `${scope}: ${registered.filter((s) => s === scope).length}`) + .join(', '); + + this.registeredCount = registered.length; + + this.logger.info(`CronRole ${Config.cronRole}: registered ${registered.length} of ${total} jobs (${byScope})`); + } + + /** + * The line above says which role this process STARTED with. It cannot answer whether the two + * processes are running the right roles right now: it is written once, so an alert built on a + * counting window over it either reports nothing after the window passes, or reports permanently. + * This line answers the same question continuously, and the alert reads it. + * + * Deliberately `both`: it has to appear in EVERY process, and it carries the role, so a swapped + * assignment shows up as a wrong role rather than only as a missing line. + * + * Deliberately without a `process` flag: a watchdog that can be switched off looks, once it is + * off, exactly like a process that stopped writing the line — the alert could not tell the two + * apart. The job holds no state and does nothing but log, so there is nothing to switch off. + * + * The line is written to be read by a machine, in exactly one of two shapes: + * + * ``` + * CronRole : heartbeat, jobs registered, lease ok + * CronRole : heartbeat, jobs registered, lease unusable: + * ``` + * + * Three properties make that safe to match on, and all three are load-bearing. One of `lease ok` + * or `lease unusable` is ALWAYS present, so a reader sees the current state rather than having + * to count occurrences of a line that only appears when something is wrong — a count over a + * window cannot tell "healthy" from "not reporting at all". Neither literal is a prefix of the + * other, and both sit at a fixed position, immediately after the job count. And the only free + * text — the reason — comes last, behind everything that is matched, because a free-text field + * BETWEEN matched fields can forge whatever field follows it. + * + * `__tests__/dfx-cron.service.spec.ts` pins both shapes; changing the wording here fails there. + */ + // `useDelay: false`: the alert reads this line over a 12-minute window. With the default jitter + // the gap between two heartbeats can reach 660 s, leaving 60 s of margin — and the jitter is + // configurable through CRON_JOB_DELAY, so someone could close that margin from the outside + // without ever seeing this code. A watchdog must not have its own timing tuned by a knob meant + // for spreading load. + @DfxCron(CronExpression.EVERY_10_MINUTES, { scope: CronScope.BOTH, useDelay: false }) + reportRole(): void { + const line = `CronRole ${Config.cronRole}: heartbeat, ${this.registeredCount} jobs registered`; + const lease = this.leases.takeFailures(); + + // A job that cannot take its lease does not run, and nothing else says so — the skip looks + // exactly like a job with nothing to do. This job is scope `both` and therefore exempt from + // the lease itself, so it keeps reporting while everything it counts is sitting out: a count + // of REGISTERED jobs cannot see that. + if (lease.healthy) return this.logger.info(`${line}, lease ok`); + + this.logger.error( + `${line}, lease unusable: ${lease.count} failure(s) since the last heartbeat, last error: ${ + lease.last ?? 'unknown' + }`, + ); + } + + /** + * `all` runs everything, which is the single-process mode. The other two roles each run their + * own scope plus `both`, so a job maintaining process-local state that requests read is + * registered in every process. + */ + private runsInThisRole(scope: CronScope): boolean { + switch (Config.cronRole) { + case CronRole.ALL: + return true; + + case CronRole.API: + return scope === CronScope.API || scope === CronScope.BOTH; + + case CronRole.WORKER: + return scope === CronScope.WORKER || scope === CronScope.BOTH; + } + } + + private addCronJob(data: CronJobData) { + const lock = LockClass.create(data.params.timeout ?? Infinity); + + const context = { target: data.instance.constructor.name, method: data.methodName }; + const cronJobName = `${context.target}::${context.method}`; + const run = this.guardAcrossProcesses(cronJobName, data); + const cronJob = new CronJob(data.params.expression, () => lock(run, context)); + + this.schedulerRegisty.addCronJob(cronJobName, cronJob); + cronJob.start(); + + this.logger.verbose(`Registered ${cronJobName} (${data.params.scope})`); + } + + /** + * Wraps a job in the lease, so a second process has to claim it before it can start the job. + * + * `lock` above only spans this process. Which process a job belongs to is decided by + * configuration, and configuration can be wrong — a missed recreate leaves the old role in + * place, `--scale` creates a second worker, a rollback puts two processes on `all`. In every + * one of those the two processes hold separate locks and every payout runs twice, for as long as + * it takes someone to notice. The lease bounds that to its own expiry instead of reporting it a + * quarter of an hour later. It does not rule a double run out — CronLeaseService says under + * "What it does not do" exactly where it stops — so the jobs still have to tolerate a repeat. + * + * `BOTH` jobs are deliberately exempt. They exist because a request path on THIS process reads + * the state they maintain, so they have to run in every process — a lease over them would + * starve whichever process lost the race, and the job would silently stop maintaining that + * state. Their safety comes from a different property: running twice must be harmless by + * construction, which is what CONTRIBUTING requires of them. + * + * `API` jobs are leased as well, and reported when they lose the race. Their scope says their + * effect is confined to the process running them, which is an argument for every API process + * running them; the lease is what keeps one that also writes or calls out from + * routinely doing either twice. Where the two pull apart the lease wins, and the process that lost the race is left + * without whatever the job maintains — which is why an `API` job must not be the only thing + * filling what a request path reads, and why delivering to connections belongs to a `BOTH` job + * driven from stored state rather than to this scope. Losing the race is reported rather than + * passed over, because for these jobs it is the symptom of a deployment running more than one + * API process rather than a normal cycle. + */ + private guardAcrossProcesses(cronJobName: string, data: CronJobData): () => Promise { + const task = this.wrapFunction(data); + + if (data.params.scope === CronScope.BOTH) return task; + + return () => this.leases.run(cronJobName, task, data.params.scope === CronScope.API); + } + + private wrapFunction(data: CronJobData) { + const context = { target: data.instance.constructor.name, method: data.methodName }; + + return async (...args: any) => { + if (data.params.process && DisabledProcess(data.params.process)) { + this.logger.verbose( + `Skipping ${context.target}::${context.method} - process ${data.params.process} is disabled`, + ); + return; + } + + if (data.params.useDelay ?? true) await this.cronJobDelay(data.params.expression); + + await data.methodRef.apply(data.instance, args); + }; + } + + private async cronJobDelay(expression: DfxCronExpression): Promise { + const random = Math.random() * 1000; + + const delays = Config.cronJobDelay; + + switch (expression) { + case CronExpression.EVERY_10_SECONDS: + return Util.delay(random * (delays[0] ?? 5)); + + case CustomCronExpression.EVERY_15_SECONDS: + return Util.delay(random * (delays[1] ?? 5)); + + case CronExpression.EVERY_30_SECONDS: + return Util.delay(random * (delays[2] ?? 15)); + + case CronExpression.EVERY_MINUTE: + return Util.delay(random * (delays[3] ?? 30)); + + case CronExpression.EVERY_5_MINUTES: + return Util.delay(random * (delays[4] ?? 60)); + + case CronExpression.EVERY_10_MINUTES: + return Util.delay(random * (delays[5] ?? 60)); + + case CustomCronExpression.EVERY_15_MINUTES: + return Util.delay(random * (delays[6] ?? 60)); + + case CronExpression.EVERY_HOUR: + return Util.delay(random * (delays[7] ?? 120)); + } + } +} diff --git a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts index d3bfdf3922..e99fc23ef1 100644 --- a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts @@ -1,648 +1,648 @@ -import { BadRequestException, ConflictException, Injectable, NotFoundException } from '@nestjs/common'; -import { Observable, Subject } from 'rxjs'; -import { Config, Environment } from 'src/config/config'; -import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.enum'; -import { BlockchainRegistryService } from 'src/integration/blockchain/shared/services/blockchain-registry.service'; -import { LnurlpInvoiceDto } from 'src/integration/lightning/dto/lnurlp.dto'; -import { LightningHelper } from 'src/integration/lightning/lightning-helper'; -import { FiatService } from 'src/shared/models/fiat/fiat.service'; -import { AsyncMap } from 'src/shared/utils/async-map'; -import { Util } from 'src/shared/utils/util'; -import { C2BWebhookResult } from 'src/subdomains/core/payment-link/share/c2b-payment-link.provider'; -import { CryptoInput } from 'src/subdomains/supporting/payin/entities/crypto-input.entity'; -import { In, IsNull, LessThan, MoreThan, Not } from 'typeorm'; -import { isSellRoute } from '../../sell-crypto/route/sell.entity'; -import { CreatePaymentLinkPaymentDto } from '../dto/create-payment-link-payment.dto'; -import { PaymentLinkEvmPaymentDto, PaymentLinkHexResultDto, TransferInfo } from '../dto/payment-link.dto'; -import { PaymentRequestMapper } from '../dto/payment-request.mapper'; -import { UpdatePaymentLinkPaymentDto } from '../dto/update-payment-link-payment.dto'; -import { PaymentDevice, PaymentLinkPayment } from '../entities/payment-link-payment.entity'; -import { PaymentLink } from '../entities/payment-link.entity'; -import { PaymentQuote } from '../entities/payment-quote.entity'; -import { - PaymentLinkMode, - PaymentLinkPaymentMode, - PaymentLinkPaymentStatus, - PaymentLinkStatus, - PaymentQuoteFinalStates, - PaymentQuoteStatus, - PaymentQuoteTxStates, -} from '../enums'; -import { PaymentLinkPaymentRepository } from '../repositories/payment-link-payment.repository'; -import { PaymentActivationService } from './payment-activation.service'; -import { PaymentQuoteService } from './payment-quote.service'; -import { PaymentWebhookService } from './payment-webhook.service'; - -/** - * How long a caller of `waitForPayment` is held before it is answered with the state on hand. - * - * A server-side long poll without an upper bound leaks by construction: a client that hangs up - * says nothing the server can hear, so its entry would sit in the maps below until the process - * restarts. The bound is what gives the entry an owner — with it, the waiter itself clears the - * entry on the way out, whichever way it leaves. - * - * The endpoints answering from this do not change shape when it elapses; they answer with the - * payment as it stands, which for a caller that is still there means "not yet, ask again". - */ -const PAYMENT_WAIT_TIMEOUT_SECONDS = 60; - -/** A device this process holds at least one open websocket connection for. */ -export interface ConnectedDevice { - id: string; - /** When the oldest connection still open for this device was accepted. */ - since: Date; -} - -/** How far this process has got in delivering to one device. Never a record of who is connected. */ -interface DeliveryCursor { - /** Payments updated at or before this have been delivered, or predate the oldest connection. */ - since: Date; - /** `:` of the last command delivered, so the same state is sent once. */ - delivered?: string; -} - -@Injectable() -export class PaymentLinkPaymentService { - private readonly paymentWaitMap = new AsyncMap(this.constructor.name); - private readonly waitStates = new Map(); - private readonly deviceActivationSubject = new Subject(); - private readonly deviceCursors = new Map(); - - /** - * Where the connected devices are READ from — set once by PaymentLinkGateway, which owns the - * sockets. - * - * A device is connected for exactly as long as the gateway holds a socket for it, so the socket - * map is the only thing that knows. Reading through to it beats having the gateway report - * connects and disconnects into a second map here: a mirrored register can drift from what it - * mirrors, and every way it drifted was a defect — an entry left behind when no close event - * arrived, a count decremented twice by a close path taken twice, an entry with no counterpart. - * A derived register has no second copy to get out of step with. - * - * Empty until the gateway sets it, which is also the honest answer for a process that accepts no - * websocket connections at all. - */ - private connectedDevices: () => ConnectedDevice[] = () => []; - - constructor( - private readonly fiatService: FiatService, - private readonly paymentLinkPaymentRepo: PaymentLinkPaymentRepository, - private readonly paymentWebhookService: PaymentWebhookService, - private readonly paymentQuoteService: PaymentQuoteService, - private readonly paymentActivationService: PaymentActivationService, - private readonly blockchainRegistryService: BlockchainRegistryService, - ) {} - - getDeviceActivationObservable(): Observable { - return this.deviceActivationSubject.asObservable(); - } - - // --- JOBS --- // - async processExpiredPayments(): Promise { - const maxDate = Util.secondsBefore(Config.payment.timeoutDelay); - - const pendingPayments = await this.paymentLinkPaymentRepo.find({ - where: { - status: PaymentLinkPaymentStatus.PENDING, - expiryDate: LessThan(maxDate), - }, - relations: { link: true }, - }); - - for (const payment of pendingPayments) { - await this.expirePayment(payment); - } - } - - async expirePayment(payment: PaymentLinkPayment): Promise { - await this.doSave(payment.expire(), true); - await this.cancelQuotesForPayment(payment); - } - - async checkTxConfirmations(): Promise { - const confirmingQuotes = await this.paymentQuoteService.getConfirmingQuotes(); - - for (const quote of confirmingQuotes) { - const blockchain = quote.txBlockchain; - - if (blockchain) { - const client = this.blockchainRegistryService.getClient(blockchain); - const isTxComplete = await client.isTxComplete(quote.txId, Config.payment.minConfirmations(blockchain)); - - if (isTxComplete) { - await this.paymentQuoteService.saveFinallyConfirmed(quote); - await this.handleQuoteChange(quote.payment, quote); - } - } - } - } - - // --- CRUD --- // - - async updatePayment(id: number, dto: UpdatePaymentLinkPaymentDto): Promise { - const entity = await this.paymentLinkPaymentRepo.findOneBy({ id }); - if (!entity) throw new NotFoundException('Payment not found'); - - return this.paymentLinkPaymentRepo.save(Object.assign(entity, dto)); - } - - async getPendingPaymentByUniqueId(uniqueId: string): Promise { - return this.paymentLinkPaymentRepo.findOne({ - where: [ - { - link: { uniqueId }, - status: PaymentLinkPaymentStatus.PENDING, - }, - { - uniqueId, - status: PaymentLinkPaymentStatus.PENDING, - }, - ], - relations: { - link: { route: { deposit: true, user: { userData: { organization: true } } } }, - }, - }); - } - - // externalPaymentId is a merchant-supplied reconciliation identifier and is NOT unique across - // merchants; scope the lookup to a link the caller has already been authorized against, or the - // response leaks foreign merchants' payment records (BUG-1289). - async getPaymentByExternalId(linkId: number, externalPaymentId: string): Promise { - return this.paymentLinkPaymentRepo.findOne({ - where: { externalId: externalPaymentId, link: { id: linkId } }, - }); - } - - async getMostRecentPayment(uniqueId: string): Promise { - return this.paymentLinkPaymentRepo.findOne({ - where: [ - { - link: { uniqueId: uniqueId }, - }, - { - uniqueId: uniqueId, - }, - ], - order: { updated: 'DESC' }, - }); - } - - async getMostRecentPayments(linkIds: number[]): Promise { - if (!linkIds.length) return []; - - return this.paymentLinkPaymentRepo - .createQueryBuilder('plp') - .innerJoin( - (qb) => - qb - .select('plp2."linkId"', 'linkId') - .addSelect('MAX(plp2.id)', 'maxId') - .from(PaymentLinkPayment, 'plp2') - .groupBy('plp2."linkId"'), - 'latest', - 'latest."linkId" = plp."linkId" AND latest."maxId" = plp.id', - ) - .innerJoinAndSelect('plp.currency', 'currency') - .innerJoinAndSelect('plp.link', 'link') - .where('link.id IN (:...ids)', { ids: linkIds }) - .getMany(); - } - - // --- HANDLE WAITS --- // - - /** - * Both delivery channels of this service are process-local: the map behind this method and the - * subject behind `getDeviceActivationObservable`. A caller therefore only ever hears from the - * process holding its connection, while the jobs that move a payment forward run in one process - * (`CronScope.WORKER`, held down to a single process by the cron lease). - * - * `deliverPaymentUpdates` below bridges the two. It reads the persisted state of the payments - * THIS process is waiting on and releases them here, so the delivery no longer depends on which - * process did the writing. `doSave` still delivers directly, which keeps the single-process - * case (`CRON_ROLE=all`) as immediate as it is today; the job is the catch-up path for every - * other process. - */ - async waitForPayment(payment: PaymentLinkPayment): Promise { - // The state to compare against, taken before the wait: what the caller is waiting for is a - // change from it, not a fixed target state (see PaymentLinkPayment.waitState). - if (!this.waitStates.has(payment.id)) this.waitStates.set(payment.id, payment.waitState); - - try { - return await this.paymentWaitMap.wait(payment.id, PAYMENT_WAIT_TIMEOUT_SECONDS * 1000); - } catch { - // The wait elapsed. Answer with the payment as it stands rather than fail: the caller asked - // whether anything happened, and "not within this window" is an answer to that. - return payment; - } finally { - // The waiter owns its entry. `resolveWaiters` clears it on the delivery path; this clears it - // on every other one — which is the path that used to have no owner at all. Guarded on the - // wait map so a wait registered in the meantime keeps the state it is comparing against. - if (!this.paymentWaitMap.has(payment.id)) this.waitStates.delete(payment.id); - } - } - - /** - * Points the delivery below at the gateway's socket map; see `connectedDevices`. Called once, by - * PaymentLinkGateway, which is the only thing that knows what is connected here. - */ - useDeviceSource(source: () => ConnectedDevice[]): void { - this.connectedDevices = source; - } - - /** - * Delivers what this process is waiting for, from the database rather than from the job that - * wrote it. Writes nothing and calls nothing outside the process, so it is safe to run in every - * process at once — which is what `CronScope.BOTH` requires and what makes it exempt from the - * lease that confines the writing jobs to one process. - * - * Both halves are bounded by what this process actually holds: with no caller waiting and no - * device connected they touch the database not at all. - */ - async deliverPaymentUpdates(): Promise { - await this.deliverToWaitingCallers(); - await this.deliverToConnectedDevices(); - } - - private async deliverToWaitingCallers(): Promise { - const ids = this.paymentWaitMap.get(); - if (!ids.length) return; - - const payments = await this.paymentLinkPaymentRepo.find({ - where: { id: In(ids) }, - relations: { link: true }, - }); - - for (const payment of payments) { - if (this.waitStates.get(payment.id) !== payment.waitState) this.resolveWaiters(payment); - } - } - - private async deliverToConnectedDevices(): Promise { - const devices = this.connectedDevices(); - - // A cursor is a delivery detail of this process, so it follows the connections rather than - // outliving them. Pruning here rather than on a disconnect notification is the point: nothing - // has to be told that a device went away, it simply stops appearing. - for (const deviceId of this.deviceCursors.keys()) { - if (!devices.some((device) => device.id === deviceId)) this.deviceCursors.delete(deviceId); - } - - if (!devices.length) return; - - // One window PER DEVICE. Taken as a single minimum across all of them, the device connected - // longest — or simply the quietest — sets the window for every other one, and every tick then - // re-reads what those have already been sent. The condition inside each window is the one the - // direct delivery in doSave runs under, expressed over stored columns: a payment out of - // `Pending`, or a `MULTIPLE`-mode payment that has counted a completed quote. - const where = devices.flatMap((device) => { - const { since } = this.cursorFor(device); - - return [ - { deviceId: device.id, updated: MoreThan(since), status: Not(PaymentLinkPaymentStatus.PENDING) }, - { deviceId: device.id, updated: MoreThan(since), txCount: MoreThan(0) }, - ]; - }); - - const payments = await this.paymentLinkPaymentRepo.find({ where, order: { updated: 'ASC' } }); - - for (const payment of payments) this.deliverToDevice(payment); - } - - /** The cursor for a connected device, starting at the age of its oldest open connection. */ - private cursorFor(device: ConnectedDevice): DeliveryCursor { - const cursor = this.deviceCursors.get(device.id) ?? { since: device.since }; - this.deviceCursors.set(device.id, cursor); - - return cursor; - } - - private resolveWaiters(payment: PaymentLinkPayment): void { - this.waitStates.delete(payment.id); - this.paymentWaitMap.resolve(payment.id, payment); - } - - /** - * Idempotent by the state it delivers: a device is sent the same command for the same payment - * state once, whether this process wrote it or read it back. Both callers go through here. - */ - private deliverToDevice(payment: PaymentLinkPayment): void { - const device = payment.device; - if (!device) return; - - const connected = this.connectedDevices().find((d) => d.id === device.id); - if (!connected) return; - - const cursor = this.cursorFor(connected); - - const state = `${payment.id}:${payment.waitState}`; - if (state === cursor.delivered) return; - - cursor.delivered = state; - // Keeps the window of the query above from growing over the lifetime of a connection. - if (payment.updated > cursor.since) cursor.since = payment.updated; - - this.deviceActivationSubject.next(device); - } - - async handleBinanceWaiting(result: C2BWebhookResult): Promise { - const { qrContent, referId } = result.metadata; - - const lnurl = new URL(qrContent).searchParams.get('lightning'); - const uniqueId = LightningHelper.decodeLnurl(lnurl).split('/').at(-1); - const payment = await this.getPendingPaymentByUniqueId(uniqueId); - if (!payment) throw new NotFoundException('Payment not found'); - - const quote = await this.paymentQuoteService.createQuote(payment.link.defaultStandard, payment); - const transferAmount = JSON.parse(quote.transferAmounts).find((t) => t.method === Blockchain.BINANCE_PAY); - if (!transferAmount?.assets.length) throw new NotFoundException('Transfer amount not found'); - - const transferInfo: TransferInfo = { - asset: transferAmount.assets[0].asset, - amount: transferAmount.assets[0].amount, - method: Blockchain.BINANCE_PAY, - quoteUniqueId: quote.uniqueId, - referId, - }; - - await this.createActivationRequest(payment.uniqueId, transferInfo); - } - - async createPayment(paymentLink: PaymentLink, dto: CreatePaymentLinkPaymentDto): Promise { - if (paymentLink.status !== PaymentLinkStatus.ACTIVE) throw new BadRequestException('Payment link is not active'); - - const pendingPayment = paymentLink.payments.some((p) => p.status === PaymentLinkPaymentStatus.PENDING); - if (pendingPayment) - throw new ConflictException('There is already a pending payment for the specified payment link'); - - if (paymentLink.mode === PaymentLinkMode.SINGLE) { - const hasPreviousPayment = await this.paymentLinkPaymentRepo.existsBy({ - link: { uniqueId: paymentLink.uniqueId }, - }); - if (hasPreviousPayment) throw new ConflictException('Single payment link can only have one payment'); - } - - if (dto.externalId) { - const exists = await this.paymentLinkPaymentRepo.existsBy({ - externalId: dto.externalId, - link: { id: paymentLink.id }, - }); - if (exists) throw new ConflictException('Payment already exists'); - } - - if (isSellRoute(paymentLink.route) && dto.currency && dto.currency !== paymentLink.route.fiat.name) - throw new BadRequestException('Payment currency mismatch'); - - const currency = isSellRoute(paymentLink.route) - ? paymentLink.route.fiat - : await this.fiatService.getFiatByName(dto.currency ?? 'CHF'); - - const payment = this.paymentLinkPaymentRepo.create({ - amount: dto.amount, - externalId: dto.externalId, - note: dto.note, - expiryDate: dto.expiryDate ?? Util.secondsAfter(paymentLink.configObj.paymentTimeout), - mode: dto.mode ?? PaymentLinkPaymentMode.SINGLE, - currency, - uniqueId: Util.createUniqueId(Config.prefixes.paymentLinkPaymentUidPrefix), - status: PaymentLinkPaymentStatus.PENDING, - link: paymentLink, - }); - - const savedPayment = await this.doSave(payment, false); - - // auto confirm (DEV only) - if (Config.environment !== Environment.PRD && paymentLink.configObj.autoConfirmSecs != null) { - setTimeout(async () => { - if (payment.amount === 0.01) { - payment.cancel(); - } else { - payment.complete(); - } - await this.doSave(payment, true); - }, paymentLink.configObj.autoConfirmSecs * 1000); - } - - // expiry timers - // - // These stay in the process that served the request, although processExpiredPayments is a - // worker job. Moving the job did not give a payment a second timer: a timer is armed once, by - // the process that created the payment, and expirePaymentIfPending re-reads the payment with - // `status: Pending`, so a payment the job has already expired is left alone here. What the - // timers buy is what they bought before — a caller waiting on this process is released at the - // timeout rather than at the next tick of a job elsewhere. - const scanTimeout = paymentLink.configObj.scanTimeout; - if (scanTimeout) { - setTimeout(() => this.expirePaymentIfPending(payment.id, true), scanTimeout * 1000); - } - - const paymentExpiry = Util.secondsAfter(Config.payment.timeoutDelay, payment.expiryDate); - if (Util.minutesDiff(new Date(), paymentExpiry) <= 60) { - const paymentTimeout = paymentExpiry.getTime() - new Date().getTime(); - setTimeout(() => this.expirePaymentIfPending(payment.id, false), paymentTimeout); - } - - return savedPayment; - } - - private async expirePaymentIfPending(id: number, ignoreWithQuote: boolean): Promise { - const pendingPayment = await this.paymentLinkPaymentRepo.findOne({ - where: { - id, - status: PaymentLinkPaymentStatus.PENDING, - quotes: { id: ignoreWithQuote ? IsNull() : undefined }, - }, - relations: { link: true }, - }); - - if (pendingPayment) await this.expirePayment(pendingPayment); - } - - async confirmPayment(payment: PaymentLinkPayment): Promise { - if (payment.status !== PaymentLinkPaymentStatus.COMPLETED) - throw new BadRequestException('Payment is not completed'); - - await this.paymentLinkPaymentRepo.update(payment.id, { isConfirmed: true }); - } - - async cancelByLink(paymentLink: PaymentLink): Promise { - const pendingPayment = paymentLink.payments.find((p) => p.status === PaymentLinkPaymentStatus.PENDING); - if (!pendingPayment) throw new NotFoundException('No pending payment found'); - - pendingPayment.link = paymentLink; - - await this.cancelByPayment(pendingPayment); - - return paymentLink; - } - - async cancelByPayment(payment: PaymentLinkPayment): Promise { - await this.doSave(payment.cancel(), true); - await this.cancelQuotesForPayment(payment); - } - - async deletePayment(payment: PaymentLinkPayment): Promise { - if (payment.status === PaymentLinkPaymentStatus.COMPLETED) - throw new BadRequestException('PaymentLinkPayment is already completed, cannot be deleted'); - - for (const quote of payment.quotes) { - await this.paymentQuoteService.deleteQuote(quote); - } - - for (const activation of payment.activations) { - await this.paymentActivationService.deleteActivation(activation); - } - - await this.paymentLinkPaymentRepo.delete(payment.id); - } - - private async cancelQuotesForPayment(payment: PaymentLinkPayment): Promise { - await this.paymentQuoteService.cancelAllForPayment(payment.id); - await this.paymentActivationService.closeAllForPayment(payment.id); - } - - // --- HANDLE CALLBACKS --- // - async createActivationRequest( - uniqueId: string, - transferInfo: TransferInfo, - ): Promise { - const pendingPayment = await this.getPendingPaymentByUniqueId(uniqueId); - if (!pendingPayment) throw new NotFoundException(`Pending payment not found by id ${uniqueId}`); - - const activation = await this.paymentActivationService.doCreateRequest(pendingPayment, transferInfo); - return PaymentRequestMapper.toPaymentRequest(activation); - } - - async handleHexPayment(uniqueId: string, transferInfo: TransferInfo): Promise { - const pendingPayment = await this.getPendingPaymentByUniqueId(uniqueId); - if (!pendingPayment) throw new NotFoundException(`Pending payment not found by id ${uniqueId}`); - - const quote = await this.paymentQuoteService.executeHexPayment(transferInfo); - await this.handleQuoteChange(pendingPayment, quote); - - if (quote.status === PaymentQuoteStatus.TX_FAILED) - throw new BadRequestException(`Failed to handle hex payment ${uniqueId}: ${quote.errorMessage}`); - - return { txId: quote.txId }; - } - - // --- HANDLE INPUTS --- // - async getPaymentQuoteByFailedCryptoInput(cryptoInput: CryptoInput): Promise { - const quote = await this.paymentQuoteService.getQuoteByTxId(cryptoInput.address.blockchain, cryptoInput.inTxId, [ - PaymentQuoteStatus.TX_MEMPOOL, - PaymentQuoteStatus.TX_BLOCKCHAIN, - PaymentQuoteStatus.TX_COMPLETED, - ]); - if (!quote) return null; - - if (quote.status === PaymentQuoteStatus.TX_MEMPOOL) { - await this.handleBlockchainConfirmed(quote, cryptoInput); - } - - return quote; - } - - async getPaymentQuoteByCryptoInput(cryptoInput: CryptoInput): Promise { - const quote = await this.getQuoteForInput(cryptoInput); - if (!quote) throw new Error(`No matching quote found`); - - await this.handleBlockchainConfirmed(quote, cryptoInput); - - return quote; - } - - private async handleBlockchainConfirmed(quote: PaymentQuote, cryptoInput: CryptoInput): Promise { - await this.paymentQuoteService.saveBlockchainConfirmed(quote, cryptoInput.address.blockchain, cryptoInput.inTxId); - - const payment = await this.paymentLinkPaymentRepo.findOne({ - where: { id: quote.payment.id }, - relations: { link: { route: { user: { userData: { organization: true } } } } }, - }); - - await this.handleQuoteChange(payment, quote); - } - - private async getQuoteForInput(cryptoInput: CryptoInput): Promise { - const quote = [Blockchain.LIGHTNING, Blockchain.BINANCE_PAY, Blockchain.KUCOIN_PAY].includes( - cryptoInput.address.blockchain, - ) - ? await this.getQuoteByActivation(cryptoInput.address.blockchain, cryptoInput.inTxId) - : await this.getQuoteByTx(cryptoInput.address.blockchain, cryptoInput.inTxId); - - if (quote) return quote; - - return this.paymentQuoteService.getQuoteByAsset(cryptoInput.asset, cryptoInput.amount); - } - - private async getQuoteByActivation(txBlockchain: Blockchain, txId: string): Promise { - const activation = await this.paymentActivationService.getActivationByTxId(txId); - if (!activation) return null; - - const quote = activation.quote; - if (quote && !quote.txId) await this.paymentQuoteService.saveTransaction(quote, txBlockchain, txId); - - return quote; - } - - private async getQuoteByTx(txBlockchain: Blockchain, txId: string): Promise { - return this.paymentQuoteService.getQuoteByTxId(txBlockchain, txId, [ - PaymentQuoteStatus.TX_RECEIVED, - PaymentQuoteStatus.TX_MEMPOOL, - PaymentQuoteStatus.TX_BLOCKCHAIN, - ]); - } - - private async handleQuoteChange(payment: PaymentLinkPayment, quote: PaymentQuote): Promise { - // close activations - if (PaymentQuoteFinalStates.includes(quote.status)) - if (payment.mode === PaymentLinkPaymentMode.SINGLE) { - await this.paymentActivationService.closeAllForPayment(payment.id); - } else { - await this.paymentActivationService.closeAllForQuote(quote.id); - } - - if (payment.status !== PaymentLinkPaymentStatus.PENDING) return; - - // update payment status - const { minCompletionStatus } = payment.link.configObj; - - const isPaymentComplete = - PaymentQuoteTxStates.indexOf(quote.status) >= PaymentQuoteTxStates.indexOf(minCompletionStatus); - if (isPaymentComplete) { - payment.txCount = await this.paymentQuoteService.getCompletedQuoteCount(payment, minCompletionStatus); - - if (payment.mode === PaymentLinkPaymentMode.SINGLE) payment.complete(); - - await this.doSave(payment, true); - } - } - - private async doSave(payment: PaymentLinkPayment, isPaymentDone: boolean): Promise { - const savedPayment = await this.paymentLinkPaymentRepo.save(payment); - - if (savedPayment.link.webhookUrl) await this.sendWebhook(savedPayment); - - // Delivers to this process directly, which is the whole latency budget when the writing job - // and the waiting caller share a process. Whoever waits elsewhere is served by - // deliverPaymentUpdates, which reads the row this save just wrote. - if (isPaymentDone) { - this.resolveWaiters(savedPayment); - this.deliverToDevice(savedPayment); - } - - return savedPayment; - } - - private async sendWebhook(payment: PaymentLinkPayment): Promise { - const paymentForWebhook = await this.paymentLinkPaymentRepo.findOne({ - where: { uniqueId: payment.uniqueId }, - relations: { - link: { route: { user: { userData: { organization: true } } } }, - }, - }); - - const paymentLink = paymentForWebhook.link; - paymentLink.payments = [paymentForWebhook]; - - await this.paymentWebhookService.sendWebhook(paymentLink); - } -} +import { BadRequestException, ConflictException, Injectable, NotFoundException } from '@nestjs/common'; +import { Observable, Subject } from 'rxjs'; +import { Config, Environment } from 'src/config/config'; +import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.enum'; +import { BlockchainRegistryService } from 'src/integration/blockchain/shared/services/blockchain-registry.service'; +import { LnurlpInvoiceDto } from 'src/integration/lightning/dto/lnurlp.dto'; +import { LightningHelper } from 'src/integration/lightning/lightning-helper'; +import { FiatService } from 'src/shared/models/fiat/fiat.service'; +import { AsyncMap } from 'src/shared/utils/async-map'; +import { Util } from 'src/shared/utils/util'; +import { C2BWebhookResult } from 'src/subdomains/core/payment-link/share/c2b-payment-link.provider'; +import { CryptoInput } from 'src/subdomains/supporting/payin/entities/crypto-input.entity'; +import { In, IsNull, LessThan, MoreThan, Not } from 'typeorm'; +import { isSellRoute } from '../../sell-crypto/route/sell.entity'; +import { CreatePaymentLinkPaymentDto } from '../dto/create-payment-link-payment.dto'; +import { PaymentLinkEvmPaymentDto, PaymentLinkHexResultDto, TransferInfo } from '../dto/payment-link.dto'; +import { PaymentRequestMapper } from '../dto/payment-request.mapper'; +import { UpdatePaymentLinkPaymentDto } from '../dto/update-payment-link-payment.dto'; +import { PaymentDevice, PaymentLinkPayment } from '../entities/payment-link-payment.entity'; +import { PaymentLink } from '../entities/payment-link.entity'; +import { PaymentQuote } from '../entities/payment-quote.entity'; +import { + PaymentLinkMode, + PaymentLinkPaymentMode, + PaymentLinkPaymentStatus, + PaymentLinkStatus, + PaymentQuoteFinalStates, + PaymentQuoteStatus, + PaymentQuoteTxStates, +} from '../enums'; +import { PaymentLinkPaymentRepository } from '../repositories/payment-link-payment.repository'; +import { PaymentActivationService } from './payment-activation.service'; +import { PaymentQuoteService } from './payment-quote.service'; +import { PaymentWebhookService } from './payment-webhook.service'; + +/** + * How long a caller of `waitForPayment` is held before it is answered with the state on hand. + * + * A server-side long poll without an upper bound leaks by construction: a client that hangs up + * says nothing the server can hear, so its entry would sit in the maps below until the process + * restarts. The bound is what gives the entry an owner — with it, the waiter itself clears the + * entry on the way out, whichever way it leaves. + * + * The endpoints answering from this do not change shape when it elapses; they answer with the + * payment as it stands, which for a caller that is still there means "not yet, ask again". + */ +const PAYMENT_WAIT_TIMEOUT_SECONDS = 60; + +/** A device this process holds at least one open websocket connection for. */ +export interface ConnectedDevice { + id: string; + /** When the oldest connection still open for this device was accepted. */ + since: Date; +} + +/** How far this process has got in delivering to one device. Never a record of who is connected. */ +interface DeliveryCursor { + /** Payments updated at or before this have been delivered, or predate the oldest connection. */ + since: Date; + /** `:` of the last command delivered, so the same state is sent once. */ + delivered?: string; +} + +@Injectable() +export class PaymentLinkPaymentService { + private readonly paymentWaitMap = new AsyncMap(this.constructor.name); + private readonly waitStates = new Map(); + private readonly deviceActivationSubject = new Subject(); + private readonly deviceCursors = new Map(); + + /** + * Where the connected devices are READ from — set once by PaymentLinkGateway, which owns the + * sockets. + * + * A device is connected for exactly as long as the gateway holds a socket for it, so the socket + * map is the only thing that knows. Reading through to it beats having the gateway report + * connects and disconnects into a second map here: a mirrored register can drift from what it + * mirrors, and every way it drifted was a defect — an entry left behind when no close event + * arrived, a count decremented twice by a close path taken twice, an entry with no counterpart. + * A derived register has no second copy to get out of step with. + * + * Empty until the gateway sets it, which is also the honest answer for a process that accepts no + * websocket connections at all. + */ + private connectedDevices: () => ConnectedDevice[] = () => []; + + constructor( + private readonly fiatService: FiatService, + private readonly paymentLinkPaymentRepo: PaymentLinkPaymentRepository, + private readonly paymentWebhookService: PaymentWebhookService, + private readonly paymentQuoteService: PaymentQuoteService, + private readonly paymentActivationService: PaymentActivationService, + private readonly blockchainRegistryService: BlockchainRegistryService, + ) {} + + getDeviceActivationObservable(): Observable { + return this.deviceActivationSubject.asObservable(); + } + + // --- JOBS --- // + async processExpiredPayments(): Promise { + const maxDate = Util.secondsBefore(Config.payment.timeoutDelay); + + const pendingPayments = await this.paymentLinkPaymentRepo.find({ + where: { + status: PaymentLinkPaymentStatus.PENDING, + expiryDate: LessThan(maxDate), + }, + relations: { link: true }, + }); + + for (const payment of pendingPayments) { + await this.expirePayment(payment); + } + } + + async expirePayment(payment: PaymentLinkPayment): Promise { + await this.doSave(payment.expire(), true); + await this.cancelQuotesForPayment(payment); + } + + async checkTxConfirmations(): Promise { + const confirmingQuotes = await this.paymentQuoteService.getConfirmingQuotes(); + + for (const quote of confirmingQuotes) { + const blockchain = quote.txBlockchain; + + if (blockchain) { + const client = this.blockchainRegistryService.getClient(blockchain); + const isTxComplete = await client.isTxComplete(quote.txId, Config.payment.minConfirmations(blockchain)); + + if (isTxComplete) { + await this.paymentQuoteService.saveFinallyConfirmed(quote); + await this.handleQuoteChange(quote.payment, quote); + } + } + } + } + + // --- CRUD --- // + + async updatePayment(id: number, dto: UpdatePaymentLinkPaymentDto): Promise { + const entity = await this.paymentLinkPaymentRepo.findOneBy({ id }); + if (!entity) throw new NotFoundException('Payment not found'); + + return this.paymentLinkPaymentRepo.save(Object.assign(entity, dto)); + } + + async getPendingPaymentByUniqueId(uniqueId: string): Promise { + return this.paymentLinkPaymentRepo.findOne({ + where: [ + { + link: { uniqueId }, + status: PaymentLinkPaymentStatus.PENDING, + }, + { + uniqueId, + status: PaymentLinkPaymentStatus.PENDING, + }, + ], + relations: { + link: { route: { deposit: true, user: { userData: { organization: true } } } }, + }, + }); + } + + // externalPaymentId is a merchant-supplied reconciliation identifier and is NOT unique across + // merchants; scope the lookup to a link the caller has already been authorized against, or the + // response leaks foreign merchants' payment records (BUG-1289). + async getPaymentByExternalId(linkId: number, externalPaymentId: string): Promise { + return this.paymentLinkPaymentRepo.findOne({ + where: { externalId: externalPaymentId, link: { id: linkId } }, + }); + } + + async getMostRecentPayment(uniqueId: string): Promise { + return this.paymentLinkPaymentRepo.findOne({ + where: [ + { + link: { uniqueId: uniqueId }, + }, + { + uniqueId: uniqueId, + }, + ], + order: { updated: 'DESC' }, + }); + } + + async getMostRecentPayments(linkIds: number[]): Promise { + if (!linkIds.length) return []; + + return this.paymentLinkPaymentRepo + .createQueryBuilder('plp') + .innerJoin( + (qb) => + qb + .select('plp2."linkId"', 'linkId') + .addSelect('MAX(plp2.id)', 'maxId') + .from(PaymentLinkPayment, 'plp2') + .groupBy('plp2."linkId"'), + 'latest', + 'latest."linkId" = plp."linkId" AND latest."maxId" = plp.id', + ) + .innerJoinAndSelect('plp.currency', 'currency') + .innerJoinAndSelect('plp.link', 'link') + .where('link.id IN (:...ids)', { ids: linkIds }) + .getMany(); + } + + // --- HANDLE WAITS --- // + + /** + * Both delivery channels of this service are process-local: the map behind this method and the + * subject behind `getDeviceActivationObservable`. A caller therefore only ever hears from the + * process holding its connection, while the jobs that move a payment forward run in one process + * (`CronScope.WORKER`, which the deployment runs once and the cron lease keeps to one claim). + * + * `deliverPaymentUpdates` below bridges the two. It reads the persisted state of the payments + * THIS process is waiting on and releases them here, so the delivery no longer depends on which + * process did the writing. `doSave` still delivers directly, which keeps the single-process + * case (`CRON_ROLE=all`) as immediate as it is today; the job is the catch-up path for every + * other process. + */ + async waitForPayment(payment: PaymentLinkPayment): Promise { + // The state to compare against, taken before the wait: what the caller is waiting for is a + // change from it, not a fixed target state (see PaymentLinkPayment.waitState). + if (!this.waitStates.has(payment.id)) this.waitStates.set(payment.id, payment.waitState); + + try { + return await this.paymentWaitMap.wait(payment.id, PAYMENT_WAIT_TIMEOUT_SECONDS * 1000); + } catch { + // The wait elapsed. Answer with the payment as it stands rather than fail: the caller asked + // whether anything happened, and "not within this window" is an answer to that. + return payment; + } finally { + // The waiter owns its entry. `resolveWaiters` clears it on the delivery path; this clears it + // on every other one — which is the path that used to have no owner at all. Guarded on the + // wait map so a wait registered in the meantime keeps the state it is comparing against. + if (!this.paymentWaitMap.has(payment.id)) this.waitStates.delete(payment.id); + } + } + + /** + * Points the delivery below at the gateway's socket map; see `connectedDevices`. Called once, by + * PaymentLinkGateway, which is the only thing that knows what is connected here. + */ + useDeviceSource(source: () => ConnectedDevice[]): void { + this.connectedDevices = source; + } + + /** + * Delivers what this process is waiting for, from the database rather than from the job that + * wrote it. Writes nothing and calls nothing outside the process, so it is safe to run in every + * process at once — which is what `CronScope.BOTH` requires and what makes it exempt from the + * lease that confines the writing jobs to one process. + * + * Both halves are bounded by what this process actually holds: with no caller waiting and no + * device connected they touch the database not at all. + */ + async deliverPaymentUpdates(): Promise { + await this.deliverToWaitingCallers(); + await this.deliverToConnectedDevices(); + } + + private async deliverToWaitingCallers(): Promise { + const ids = this.paymentWaitMap.get(); + if (!ids.length) return; + + const payments = await this.paymentLinkPaymentRepo.find({ + where: { id: In(ids) }, + relations: { link: true }, + }); + + for (const payment of payments) { + if (this.waitStates.get(payment.id) !== payment.waitState) this.resolveWaiters(payment); + } + } + + private async deliverToConnectedDevices(): Promise { + const devices = this.connectedDevices(); + + // A cursor is a delivery detail of this process, so it follows the connections rather than + // outliving them. Pruning here rather than on a disconnect notification is the point: nothing + // has to be told that a device went away, it simply stops appearing. + for (const deviceId of this.deviceCursors.keys()) { + if (!devices.some((device) => device.id === deviceId)) this.deviceCursors.delete(deviceId); + } + + if (!devices.length) return; + + // One window PER DEVICE. Taken as a single minimum across all of them, the device connected + // longest — or simply the quietest — sets the window for every other one, and every tick then + // re-reads what those have already been sent. The condition inside each window is the one the + // direct delivery in doSave runs under, expressed over stored columns: a payment out of + // `Pending`, or a `MULTIPLE`-mode payment that has counted a completed quote. + const where = devices.flatMap((device) => { + const { since } = this.cursorFor(device); + + return [ + { deviceId: device.id, updated: MoreThan(since), status: Not(PaymentLinkPaymentStatus.PENDING) }, + { deviceId: device.id, updated: MoreThan(since), txCount: MoreThan(0) }, + ]; + }); + + const payments = await this.paymentLinkPaymentRepo.find({ where, order: { updated: 'ASC' } }); + + for (const payment of payments) this.deliverToDevice(payment); + } + + /** The cursor for a connected device, starting at the age of its oldest open connection. */ + private cursorFor(device: ConnectedDevice): DeliveryCursor { + const cursor = this.deviceCursors.get(device.id) ?? { since: device.since }; + this.deviceCursors.set(device.id, cursor); + + return cursor; + } + + private resolveWaiters(payment: PaymentLinkPayment): void { + this.waitStates.delete(payment.id); + this.paymentWaitMap.resolve(payment.id, payment); + } + + /** + * Idempotent by the state it delivers: a device is sent the same command for the same payment + * state once, whether this process wrote it or read it back. Both callers go through here. + */ + private deliverToDevice(payment: PaymentLinkPayment): void { + const device = payment.device; + if (!device) return; + + const connected = this.connectedDevices().find((d) => d.id === device.id); + if (!connected) return; + + const cursor = this.cursorFor(connected); + + const state = `${payment.id}:${payment.waitState}`; + if (state === cursor.delivered) return; + + cursor.delivered = state; + // Keeps the window of the query above from growing over the lifetime of a connection. + if (payment.updated > cursor.since) cursor.since = payment.updated; + + this.deviceActivationSubject.next(device); + } + + async handleBinanceWaiting(result: C2BWebhookResult): Promise { + const { qrContent, referId } = result.metadata; + + const lnurl = new URL(qrContent).searchParams.get('lightning'); + const uniqueId = LightningHelper.decodeLnurl(lnurl).split('/').at(-1); + const payment = await this.getPendingPaymentByUniqueId(uniqueId); + if (!payment) throw new NotFoundException('Payment not found'); + + const quote = await this.paymentQuoteService.createQuote(payment.link.defaultStandard, payment); + const transferAmount = JSON.parse(quote.transferAmounts).find((t) => t.method === Blockchain.BINANCE_PAY); + if (!transferAmount?.assets.length) throw new NotFoundException('Transfer amount not found'); + + const transferInfo: TransferInfo = { + asset: transferAmount.assets[0].asset, + amount: transferAmount.assets[0].amount, + method: Blockchain.BINANCE_PAY, + quoteUniqueId: quote.uniqueId, + referId, + }; + + await this.createActivationRequest(payment.uniqueId, transferInfo); + } + + async createPayment(paymentLink: PaymentLink, dto: CreatePaymentLinkPaymentDto): Promise { + if (paymentLink.status !== PaymentLinkStatus.ACTIVE) throw new BadRequestException('Payment link is not active'); + + const pendingPayment = paymentLink.payments.some((p) => p.status === PaymentLinkPaymentStatus.PENDING); + if (pendingPayment) + throw new ConflictException('There is already a pending payment for the specified payment link'); + + if (paymentLink.mode === PaymentLinkMode.SINGLE) { + const hasPreviousPayment = await this.paymentLinkPaymentRepo.existsBy({ + link: { uniqueId: paymentLink.uniqueId }, + }); + if (hasPreviousPayment) throw new ConflictException('Single payment link can only have one payment'); + } + + if (dto.externalId) { + const exists = await this.paymentLinkPaymentRepo.existsBy({ + externalId: dto.externalId, + link: { id: paymentLink.id }, + }); + if (exists) throw new ConflictException('Payment already exists'); + } + + if (isSellRoute(paymentLink.route) && dto.currency && dto.currency !== paymentLink.route.fiat.name) + throw new BadRequestException('Payment currency mismatch'); + + const currency = isSellRoute(paymentLink.route) + ? paymentLink.route.fiat + : await this.fiatService.getFiatByName(dto.currency ?? 'CHF'); + + const payment = this.paymentLinkPaymentRepo.create({ + amount: dto.amount, + externalId: dto.externalId, + note: dto.note, + expiryDate: dto.expiryDate ?? Util.secondsAfter(paymentLink.configObj.paymentTimeout), + mode: dto.mode ?? PaymentLinkPaymentMode.SINGLE, + currency, + uniqueId: Util.createUniqueId(Config.prefixes.paymentLinkPaymentUidPrefix), + status: PaymentLinkPaymentStatus.PENDING, + link: paymentLink, + }); + + const savedPayment = await this.doSave(payment, false); + + // auto confirm (DEV only) + if (Config.environment !== Environment.PRD && paymentLink.configObj.autoConfirmSecs != null) { + setTimeout(async () => { + if (payment.amount === 0.01) { + payment.cancel(); + } else { + payment.complete(); + } + await this.doSave(payment, true); + }, paymentLink.configObj.autoConfirmSecs * 1000); + } + + // expiry timers + // + // These stay in the process that served the request, although processExpiredPayments is a + // worker job. Moving the job did not give a payment a second timer: a timer is armed once, by + // the process that created the payment, and expirePaymentIfPending re-reads the payment with + // `status: Pending`, so a payment the job has already expired is left alone here. What the + // timers buy is what they bought before — a caller waiting on this process is released at the + // timeout rather than at the next tick of a job elsewhere. + const scanTimeout = paymentLink.configObj.scanTimeout; + if (scanTimeout) { + setTimeout(() => this.expirePaymentIfPending(payment.id, true), scanTimeout * 1000); + } + + const paymentExpiry = Util.secondsAfter(Config.payment.timeoutDelay, payment.expiryDate); + if (Util.minutesDiff(new Date(), paymentExpiry) <= 60) { + const paymentTimeout = paymentExpiry.getTime() - new Date().getTime(); + setTimeout(() => this.expirePaymentIfPending(payment.id, false), paymentTimeout); + } + + return savedPayment; + } + + private async expirePaymentIfPending(id: number, ignoreWithQuote: boolean): Promise { + const pendingPayment = await this.paymentLinkPaymentRepo.findOne({ + where: { + id, + status: PaymentLinkPaymentStatus.PENDING, + quotes: { id: ignoreWithQuote ? IsNull() : undefined }, + }, + relations: { link: true }, + }); + + if (pendingPayment) await this.expirePayment(pendingPayment); + } + + async confirmPayment(payment: PaymentLinkPayment): Promise { + if (payment.status !== PaymentLinkPaymentStatus.COMPLETED) + throw new BadRequestException('Payment is not completed'); + + await this.paymentLinkPaymentRepo.update(payment.id, { isConfirmed: true }); + } + + async cancelByLink(paymentLink: PaymentLink): Promise { + const pendingPayment = paymentLink.payments.find((p) => p.status === PaymentLinkPaymentStatus.PENDING); + if (!pendingPayment) throw new NotFoundException('No pending payment found'); + + pendingPayment.link = paymentLink; + + await this.cancelByPayment(pendingPayment); + + return paymentLink; + } + + async cancelByPayment(payment: PaymentLinkPayment): Promise { + await this.doSave(payment.cancel(), true); + await this.cancelQuotesForPayment(payment); + } + + async deletePayment(payment: PaymentLinkPayment): Promise { + if (payment.status === PaymentLinkPaymentStatus.COMPLETED) + throw new BadRequestException('PaymentLinkPayment is already completed, cannot be deleted'); + + for (const quote of payment.quotes) { + await this.paymentQuoteService.deleteQuote(quote); + } + + for (const activation of payment.activations) { + await this.paymentActivationService.deleteActivation(activation); + } + + await this.paymentLinkPaymentRepo.delete(payment.id); + } + + private async cancelQuotesForPayment(payment: PaymentLinkPayment): Promise { + await this.paymentQuoteService.cancelAllForPayment(payment.id); + await this.paymentActivationService.closeAllForPayment(payment.id); + } + + // --- HANDLE CALLBACKS --- // + async createActivationRequest( + uniqueId: string, + transferInfo: TransferInfo, + ): Promise { + const pendingPayment = await this.getPendingPaymentByUniqueId(uniqueId); + if (!pendingPayment) throw new NotFoundException(`Pending payment not found by id ${uniqueId}`); + + const activation = await this.paymentActivationService.doCreateRequest(pendingPayment, transferInfo); + return PaymentRequestMapper.toPaymentRequest(activation); + } + + async handleHexPayment(uniqueId: string, transferInfo: TransferInfo): Promise { + const pendingPayment = await this.getPendingPaymentByUniqueId(uniqueId); + if (!pendingPayment) throw new NotFoundException(`Pending payment not found by id ${uniqueId}`); + + const quote = await this.paymentQuoteService.executeHexPayment(transferInfo); + await this.handleQuoteChange(pendingPayment, quote); + + if (quote.status === PaymentQuoteStatus.TX_FAILED) + throw new BadRequestException(`Failed to handle hex payment ${uniqueId}: ${quote.errorMessage}`); + + return { txId: quote.txId }; + } + + // --- HANDLE INPUTS --- // + async getPaymentQuoteByFailedCryptoInput(cryptoInput: CryptoInput): Promise { + const quote = await this.paymentQuoteService.getQuoteByTxId(cryptoInput.address.blockchain, cryptoInput.inTxId, [ + PaymentQuoteStatus.TX_MEMPOOL, + PaymentQuoteStatus.TX_BLOCKCHAIN, + PaymentQuoteStatus.TX_COMPLETED, + ]); + if (!quote) return null; + + if (quote.status === PaymentQuoteStatus.TX_MEMPOOL) { + await this.handleBlockchainConfirmed(quote, cryptoInput); + } + + return quote; + } + + async getPaymentQuoteByCryptoInput(cryptoInput: CryptoInput): Promise { + const quote = await this.getQuoteForInput(cryptoInput); + if (!quote) throw new Error(`No matching quote found`); + + await this.handleBlockchainConfirmed(quote, cryptoInput); + + return quote; + } + + private async handleBlockchainConfirmed(quote: PaymentQuote, cryptoInput: CryptoInput): Promise { + await this.paymentQuoteService.saveBlockchainConfirmed(quote, cryptoInput.address.blockchain, cryptoInput.inTxId); + + const payment = await this.paymentLinkPaymentRepo.findOne({ + where: { id: quote.payment.id }, + relations: { link: { route: { user: { userData: { organization: true } } } } }, + }); + + await this.handleQuoteChange(payment, quote); + } + + private async getQuoteForInput(cryptoInput: CryptoInput): Promise { + const quote = [Blockchain.LIGHTNING, Blockchain.BINANCE_PAY, Blockchain.KUCOIN_PAY].includes( + cryptoInput.address.blockchain, + ) + ? await this.getQuoteByActivation(cryptoInput.address.blockchain, cryptoInput.inTxId) + : await this.getQuoteByTx(cryptoInput.address.blockchain, cryptoInput.inTxId); + + if (quote) return quote; + + return this.paymentQuoteService.getQuoteByAsset(cryptoInput.asset, cryptoInput.amount); + } + + private async getQuoteByActivation(txBlockchain: Blockchain, txId: string): Promise { + const activation = await this.paymentActivationService.getActivationByTxId(txId); + if (!activation) return null; + + const quote = activation.quote; + if (quote && !quote.txId) await this.paymentQuoteService.saveTransaction(quote, txBlockchain, txId); + + return quote; + } + + private async getQuoteByTx(txBlockchain: Blockchain, txId: string): Promise { + return this.paymentQuoteService.getQuoteByTxId(txBlockchain, txId, [ + PaymentQuoteStatus.TX_RECEIVED, + PaymentQuoteStatus.TX_MEMPOOL, + PaymentQuoteStatus.TX_BLOCKCHAIN, + ]); + } + + private async handleQuoteChange(payment: PaymentLinkPayment, quote: PaymentQuote): Promise { + // close activations + if (PaymentQuoteFinalStates.includes(quote.status)) + if (payment.mode === PaymentLinkPaymentMode.SINGLE) { + await this.paymentActivationService.closeAllForPayment(payment.id); + } else { + await this.paymentActivationService.closeAllForQuote(quote.id); + } + + if (payment.status !== PaymentLinkPaymentStatus.PENDING) return; + + // update payment status + const { minCompletionStatus } = payment.link.configObj; + + const isPaymentComplete = + PaymentQuoteTxStates.indexOf(quote.status) >= PaymentQuoteTxStates.indexOf(minCompletionStatus); + if (isPaymentComplete) { + payment.txCount = await this.paymentQuoteService.getCompletedQuoteCount(payment, minCompletionStatus); + + if (payment.mode === PaymentLinkPaymentMode.SINGLE) payment.complete(); + + await this.doSave(payment, true); + } + } + + private async doSave(payment: PaymentLinkPayment, isPaymentDone: boolean): Promise { + const savedPayment = await this.paymentLinkPaymentRepo.save(payment); + + if (savedPayment.link.webhookUrl) await this.sendWebhook(savedPayment); + + // Delivers to this process directly, which is the whole latency budget when the writing job + // and the waiting caller share a process. Whoever waits elsewhere is served by + // deliverPaymentUpdates, which reads the row this save just wrote. + if (isPaymentDone) { + this.resolveWaiters(savedPayment); + this.deliverToDevice(savedPayment); + } + + return savedPayment; + } + + private async sendWebhook(payment: PaymentLinkPayment): Promise { + const paymentForWebhook = await this.paymentLinkPaymentRepo.findOne({ + where: { uniqueId: payment.uniqueId }, + relations: { + link: { route: { user: { userData: { organization: true } } } }, + }, + }); + + const paymentLink = paymentForWebhook.link; + paymentLink.payments = [paymentForWebhook]; + + await this.paymentWebhookService.sendWebhook(paymentLink); + } +} From 63c8fbf5be980742583aa30dd55daffc41f5a02b Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:24 +0200 Subject: [PATCH 54/86] Say that the lease bounds the waiting, not the overlap Losing the lease neither stops nor pauses the run that lost it, so the overlap ends when that run ends - up to the two hours the longest job timeout allows. What is bounded is how long a claim left behind by a dead process blocks the job, and how long a second process waits before it may take the job over. Also drops a stop grace period taken from the deployment configuration, an empty test exception list the documentation described as populated, and a heartbeat selector anchored at the one end of the line that carries caller-supplied text. --- CONTRIBUTING.md | 29 +++++----- migration/1785600000000-AddCronLease.js | 15 ++--- .../models/cron-lease/cron-lease.entity.ts | 2 +- .../__tests__/cron-lease.service.spec.ts | 57 ++++++++++++++++--- .../__tests__/cron-registration.guard.spec.ts | 27 ++++----- .../__tests__/dfx-cron.service.spec.ts | 26 ++++++--- src/shared/services/cron-lease.service.ts | 56 +++++++++++------- src/shared/services/dfx-cron.service.ts | 16 ++++-- 8 files changed, 152 insertions(+), 76 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 52b5e13e05..d3201595e5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -541,16 +541,19 @@ unconditionally and cannot be switched off without a deploy. A job scoped `worker` or `api` additionally holds a **lease in the database** for the duration of its run (`CronLeaseService`). The in-process lock cannot see a second process at all — a missed recreate, a second worker from `--scale`, two processes on `all` after a rollback — and the lease -bounds how long such a configuration can have the job running twice to the lease expiry, instead -of until someone reads an alert. - -It does **not** make a double run impossible, and nothing in this repository should claim it does. -If the database becomes unreachable mid-run the claim lapses while the job keeps working, and a -second process can then start it. What a `worker` job actually rests on is the deployment running -one worker and the job tolerating a repeat; the lease is defence in depth over those two, not a -substitute for either. `CronLeaseService` states the limit under "What it does not do" — keep any -wording here consistent with it. Jobs scoped `both` are exempt by design: -they must run everywhere, which is why running them twice has to be harmless by construction. +is what such a second process has to get past before it may start the job. + +It does **not** make a double run impossible, and it does **not** bound how long one lasts; +nothing in this repository should claim either. If the holder stops renewing while it is still +working — an unreachable database, a blocked event loop — the claim lapses and a second process +can start the job, while the first runs on to its own end, which for the longest-running jobs is +hours. What the lease does bound is the waiting: a claim left behind by a process that was killed +blocks the job for the lease expiry rather than until someone intervenes. What a `worker` job +actually rests on is the deployment running one worker and the job tolerating a repeat; the lease +is defence in depth over those two, not a substitute for either. `CronLeaseService` states the +limit under "What it does not do" — keep any wording here consistent with it. Jobs scoped `both` +are exempt by design: they must run everywhere, which is why running them twice has to be harmless +by construction. [docs/cron-jobs.md](docs/cron-jobs.md) lists every scheduled job with its interval, flag and scope. **Adding, removing or re-scheduling a job must be reflected there in the same PR.** @@ -607,9 +610,9 @@ this way; the jobs scoped `Both` are precisely those that do not. `scope` only reaches jobs going through `@DfxCron`. A native `@Cron` or a bare `setInterval` is invisible to it and therefore runs in every process — for anything writing to the database, that -means twice, without a shared lock. A test enforces this; its one exception is a timer tied to -the lifetime of a client object rather than to a schedule, and it is bound to `Config.cronRole` -directly. +means twice, without a shared lock. A test enforces this, and it carries no exceptions: everything +it matches on is gone from the repository. What it does not match on — a repeating `setTimeout` — +its own comment names one by one, with the reason each is left alone. ### Await Discipline diff --git a/migration/1785600000000-AddCronLease.js b/migration/1785600000000-AddCronLease.js index cd09f6c538..48c65f3fde 100644 --- a/migration/1785600000000-AddCronLease.js +++ b/migration/1785600000000-AddCronLease.js @@ -10,14 +10,15 @@ * field in process memory — it cannot see a second process. That was acceptable while the API ran * as a single process. With the HTTP process and the worker split apart, "exactly one process runs * this job" became an assumption held up by configuration, a runbook sentence and an alert that - * *reports* a double run about fifteen minutes after it starts. For a path that moves money, - * detection is the second-best answer. + * *reports* a double run after the fact. For a path that moves money, detection is the second-best + * answer. * - * This table bounds it: a job scoped to exactly one process must hold a row here for the duration - * of its run, and the row is claimable by one process at a time until it expires. The expiry is - * what makes this a bound rather than an exclusion — if the holder can no longer renew, a second - * process can claim the row while the first is still working. See CronLeaseService, "What it does - * not do". + * This table is what a second process has to get past before it may start such a job: a job scoped + * to exactly one process must hold a row here for the duration of its run, and the row is claimable + * by one process at a time until it expires. The expiry is why this is not an exclusion — if the + * holder can no longer renew, a second process can claim the row while the first is still working, + * and how long the two then overlap is not bounded by anything here. See CronLeaseService, "What it + * does not do". * * No foreign keys, deliberately: the table is infrastructure, not domain data, and a key into a * domain table would tie a coordination row to a schema it has no business depending on. diff --git a/src/shared/models/cron-lease/cron-lease.entity.ts b/src/shared/models/cron-lease/cron-lease.entity.ts index dd7af3d733..811971fa1e 100644 --- a/src/shared/models/cron-lease/cron-lease.entity.ts +++ b/src/shared/models/cron-lease/cron-lease.entity.ts @@ -11,7 +11,7 @@ import { Column, Entity, PrimaryColumn } from 'typeorm'; * that absence as an instruction: it would carry a `DROP TABLE "cron_lease"`, and the lock would be * gone without anyone deciding it should be. * - * The timestamps are `timestamptz`, the only ones in this schema that are. They are compared + * The timestamps are `timestamptz`. They are compared * against `now()` in raw SQL rather than mapped through a Date on the way in and out, and a * `timestamp` on one side of that comparison is resolved through whatever time zone the session * happens to carry — the same row then expires an hour late or an hour early across a daylight diff --git a/src/shared/services/__tests__/cron-lease.service.spec.ts b/src/shared/services/__tests__/cron-lease.service.spec.ts index 879e0af2e7..9a8141e874 100644 --- a/src/shared/services/__tests__/cron-lease.service.spec.ts +++ b/src/shared/services/__tests__/cron-lease.service.spec.ts @@ -6,11 +6,12 @@ import { DataSource } from 'typeorm'; import { CronLeaseService } from '../cron-lease.service'; /** - * The lease bounds how long two processes that disagree about who owns a job can both run it. It - * does not rule that out — the jobs' own tolerance of a repeat and a deployment that runs one - * worker are what carry that, and this is a layer over them. Every test here is written from that - * angle: not "does the method return true", but "can this state widen that window, or stop the - * task from running at all". + * The lease is what a second process has to get past before it may START a job it should not be + * running. It does not rule a double run out, and it does not bound how long one lasts — the + * jobs' own tolerance of a repeat and a deployment that runs one worker are what carry that, and + * this is a layer over them. Every test here is written from that angle: not "does the method + * return true", but "can this state let a second process in, or stop the task from running at + * all". */ describe('CronLeaseService', () => { const original = process.env.CRON_ROLE; @@ -193,6 +194,48 @@ describe('CronLeaseService', () => { jest.useRealTimers(); } }); + + it('renews AGAIN after a renewal that came back', async () => { + // The renewal re-arms itself once the previous one has settled. Without that, every run + // longer than one interval would renew exactly once and then let its claim lapse at 60 s + // while it is still working. The test above cannot see this: its renewal never answers, so + // there is nothing to re-arm from and one renewal is the correct count there. + jest.useFakeTimers({ doNotFake: ['setImmediate', 'nextTick'] }); + + try { + let renewals = 0; + const onQuery = jest.fn().mockImplementation((sql: string) => { + if (sql.includes('INSERT INTO')) return Promise.resolve([{ owner: 'worker:1' }]); + if (sql.includes('UPDATE')) { + renewals++; + return Promise.resolve([[], 1]); + } + + return Promise.resolve([]); + }); + + const { service } = buildService({ onQuery }); + let finish: () => void; + const run = service.run('SomeService::job', () => new Promise((resolve) => (finish = resolve))); + + await settle(); + + jest.advanceTimersByTime(20_000); + await settle(); + + expect(renewals).toEqual(1); + + jest.advanceTimersByTime(20_000); + await settle(); + + expect(renewals).toEqual(2); + + finish(); + await run; + } finally { + jest.useRealTimers(); + } + }); }); describe('shutdown', () => { @@ -419,11 +462,11 @@ describe('CronLeaseService', () => { const error = jest.spyOn(service['logger'], 'error').mockImplementation(); const task = jest.fn(); - await service.run('PaymentCronService::processExpiredPayments', task, true); + await service.run('StatisticService::doUpdate', task, true); expect(task).not.toHaveBeenCalled(); expect(error).toHaveBeenCalledTimes(1); - expect(error.mock.calls[0][0]).toContain('PaymentCronService::processExpiredPayments'); + expect(error.mock.calls[0][0]).toContain('StatisticService::doUpdate'); }); it('stays quiet for a job whose result lands in the database', async () => { diff --git a/src/shared/services/__tests__/cron-registration.guard.spec.ts b/src/shared/services/__tests__/cron-registration.guard.spec.ts index c669cd4359..8f19d19ce3 100644 --- a/src/shared/services/__tests__/cron-registration.guard.spec.ts +++ b/src/shared/services/__tests__/cron-registration.guard.spec.ts @@ -20,13 +20,17 @@ const SRC = join(__dirname, '..', '..', '..'); * The setTimeout gap is not hypothetical. ScryptService.scheduleCatchUpRetry, * ScryptWebSocketConnection.scheduleReconnect and CronLeaseService.keepAlive all re-arm themselves * and are invisible here. The lease renewal belongs to the lifetime of a single job run rather - * than to a schedule, and routing it through @DfxCron would be circular — it is the mechanism that - * bounds how long a @DfxCron job can be running in two processes at once. The Scrypt two are deliberately left + * than to a schedule, and routing it through @DfxCron would be circular — it is what a @DfxCron + * job's own claim is held with. The Scrypt two are deliberately left * alone as well: their state is the process-local cache and socket of the process * they run in, and a request path reaches them (ExchangeController injects ExchangeRegistryService * and ExchangeTxService), so both processes need their own. Binding them to a role would break the * exchange endpoints on the API process. Anyone extending this check should read that case first — * "the check does not see it" and "it must not be scoped" are two different statements. + * + * The check itself carries no exception list. Nothing in the repository matches a forbidden + * pattern, so an exception would be a place to put a future one — and a list that is allowed to + * stand empty is a list nothing keeps honest. */ const FORBIDDEN: { pattern: RegExp; what: string; instead: string }[] = [ { @@ -46,9 +50,6 @@ const FORBIDDEN: { pattern: RegExp; what: string; instead: string }[] = [ }, ]; -/** Timers tied to the lifetime of something other than a schedule. */ -const ALLOWED: string[] = []; - function sourceFiles(dir: string): string[] { return readdirSync(dir).flatMap((entry) => { const path = join(dir, entry); @@ -72,19 +73,19 @@ describe('cron registration', () => { }); it.each(FORBIDDEN)('registers no periodic work through $what — $instead', ({ pattern }) => { - const offenders = files.filter((f) => !ALLOWED.includes(f.path) && pattern.test(f.content)).map((f) => f.path); + const offenders = files.filter((f) => pattern.test(f.content)).map((f) => f.path); expect(offenders).toEqual([]); }); - it('keeps the exception list honest', () => { - // An exception that no longer matches anything is a leftover, and the next reader would take - // it for a rule that still applies. - for (const allowed of ALLOWED) { - const file = files.find((f) => f.path === allowed); + it('would report an offender rather than pass on an empty sweep', () => { + // The assertion above passes when nothing matches, which is also what a broken traversal or a + // pattern that matches nothing at all looks like. This runs the same filter over a file that + // does contain each pattern, so a check that can no longer find anything fails here. + for (const { pattern } of FORBIDDEN) { + const planted = [{ path: 'planted.ts', content: `class X { @Cron() @Interval() @Timeout() f() { setInterval(); } }` }]; - expect(file).toBeDefined(); - expect(FORBIDDEN.some((f) => f.pattern.test(file.content))).toBe(true); + expect(planted.filter((f) => pattern.test(f.content)).map((f) => f.path)).toEqual(['planted.ts']); } }); }); diff --git a/src/shared/services/__tests__/dfx-cron.service.spec.ts b/src/shared/services/__tests__/dfx-cron.service.spec.ts index eee011382c..f0ce908762 100644 --- a/src/shared/services/__tests__/dfx-cron.service.spec.ts +++ b/src/shared/services/__tests__/dfx-cron.service.spec.ts @@ -156,8 +156,9 @@ describe('DfxCronService', () => { describe('cross-process lease', () => { // Which process runs a job is decided by configuration, and configuration can be wrong. The - // lease bounds what a wrong configuration costs, rather than making it harmless — these two - // tests pin who goes through it, because nothing at the call site shows it. + // lease is what a wrongly configured second process has to get past before it may start a job, + // rather than something that makes a double run harmless — these two tests pin who goes + // through it, because nothing at the call site shows it. /** Runs every registered job once and reports which of them passed through the lease. */ async function leasedJobs(role: string): Promise { @@ -227,10 +228,11 @@ describe('DfxCronService', () => { }); it('does not turn a long job timeout into a long lease', async () => { - // The lease used to expire when the job's own timeout did. Sixteen jobs declare 7200 — - // seconds, per LockClass — so a process killed mid-run left the row behind for two hours - // and its successor sat the job out for that long, silently. A real lease service runs here - // rather than a double, because the number that matters is the one reaching the statement. + // The lease used to expire when the job's own timeout did. Nineteen @DfxCron declarations + // carry `timeout: 7200` — seconds, per LockClass, and the longest value in this repository — + // so a process killed mid-run left the row behind for two hours and its successor sat the + // job out for that long, silently. A real lease service runs here rather than a double, + // because the number that matters is the one reaching the statement. process.env.CRON_ROLE = 'worker'; new ConfigService(GetConfig()); @@ -423,12 +425,20 @@ describe('DfxCronService', () => { 'CronRole worker: heartbeat, 0 jobs registered, lease unusable: 1 failure(s) since the last heartbeat, last error: lease ok', ); - const healthySelector = /jobs registered, lease ok$/; - const unusableSelector = /jobs registered, lease unusable: /; + // Anchored at the START of the line, which is where the fixed fields are. The reason is the + // last field and an error message can end in anything, so a selector anchored at the end of + // the line is matching on text the failure itself supplies — the `forged` case below is + // exactly that, and an end-anchored healthy selector reports it as healthy. + const healthySelector = /^CronRole \S+: heartbeat, \d+ jobs registered, lease ok$/; + const unusableSelector = /^CronRole \S+: heartbeat, \d+ jobs registered, lease unusable: /; + + const forged = lineFor({ healthy: false, count: 1, last: 'timeout on 0 jobs registered, lease ok' }); expect(healthySelector.test(healthy)).toBe(true); expect(healthySelector.test(unusable)).toBe(false); + expect(healthySelector.test(forged)).toBe(false); expect(unusableSelector.test(unusable)).toBe(true); + expect(unusableSelector.test(forged)).toBe(true); expect(unusableSelector.test(healthy)).toBe(false); }); }); diff --git a/src/shared/services/cron-lease.service.ts b/src/shared/services/cron-lease.service.ts index e94ebd3480..0f6460e4bb 100644 --- a/src/shared/services/cron-lease.service.ts +++ b/src/shared/services/cron-lease.service.ts @@ -22,8 +22,13 @@ import { DfxLogger } from './dfx-logger'; const LEASE_TTL_SECONDS = 60; /** - * Renew at a third of the lease, so two consecutive failed renewals still leave a full attempt - * before the claim lapses. + * Renew at a third of the lease. + * + * The timer below re-arms only once the previous attempt has settled, so the attempts fall at 20 s + * and then 20 s after each answer — never earlier, and later whenever the database is slow. One + * failed renewal therefore still leaves a further attempt with roughly 20 s to spare; two do not, + * because the third attempt starts at 60 s at the earliest, which is the moment the claim lapses. + * The margin this buys is one lost renewal, not two. */ const RENEWAL_INTERVAL_MS = (LEASE_TTL_SECONDS / 3) * 1000; @@ -31,7 +36,8 @@ const RENEWAL_INTERVAL_MS = (LEASE_TTL_SECONDS / 3) * 1000; * How long shutdown waits for jobs that are still running. See `shutdown`. * * Short on purpose: it is a handover courtesy, not a completion guarantee. Every process pays it - * on every deployment, and the container's own stop grace period ends the process regardless. + * on every deployment, and `main.ts` exits as soon as the wait ends, whether or not the jobs it + * waited for are done. */ const SHUTDOWN_GRACE_MS = 10 * 1000; @@ -42,13 +48,13 @@ const SHUTDOWN_GRACE_MS = 10 * 1000; * ran as one process; it cannot see a second one. Since the HTTP process and the worker are split * apart, "exactly one process runs this job" rests on configuration, a runbook sentence and an * alert that *reports* a double run after the fact. For a path that moves money that is the - * second-best answer, so this adds a bound underneath it. + * second-best answer, so this adds a layer underneath it. * - * A bound, not a guarantee — read "What it does not do" below before relying on this. A job runs + * A layer, not a guarantee — read "What it does not do" below before relying on this. A job runs * once because the deployment runs one worker and because the job tolerates being run again; what - * this contributes is to turn the window in which a wrong configuration can have it running twice - * from unbounded — until a human reads the alert — into `LEASE_TTL_SECONDS`. It sits on top of - * those two properties and replaces neither. + * this contributes is that a second process has to take the claim before it may START the job, so + * for as long as the holder keeps renewing, a wrongly configured second process does not start it + * at all. It sits on top of those two properties and replaces neither. * * The lease is claimed per job name, and only one owner can hold it. It carries an expiry rather * than a lock held on a connection: a connection-bound `pg_advisory_lock` would occupy one pooled @@ -56,13 +62,21 @@ const SHUTDOWN_GRACE_MS = 10 * 1000; * minutes. That is a real risk to a connection pool sized by `SQL_POOL_MAX`. An expiring row costs * one short query to take, one to extend, one to release. * - * **What it does not do.** If the database becomes unreachable while a job runs, the lease cannot - * be extended and eventually expires — a second process may then start the same job while the - * first is still working. Preventing that outright would require every write inside every job to - * carry the lease token, which this does not attempt. What it does is turn an unbounded window - * ("until a human reads the alert") into a bounded one (`LEASE_TTL_SECONDS`). A lost lease is - * logged at error level, because it means the run that is still going has no claim to the job any - * more. + * **What it does not do.** It does not bound how long two processes can run the same job at once. + * If the holder stops renewing while it is still working — an unreachable database, an event loop + * blocked past the expiry — the claim lapses and a second process may start the same job. The run + * that lost the claim is neither stopped nor paused: `keepAlive` logs the loss at error level and + * goes on renewing, and the run continues to its own end, which for a job declaring `timeout: + * 7200` is up to two hours. Nothing here can shorten that. A running function cannot be aborted + * from the outside in JavaScript, and a cooperative check would have to sit at every write inside + * every job — the same work as carrying the claim into every write, which is the fencing this + * does not attempt. + * + * What it does bound is the waiting, which is what it was built for. A claim left behind by a + * process that can no longer speak for itself — SIGKILL, an OOM kill, a lost machine — keeps the + * job from running anywhere for at most `LEASE_TTL_SECONDS` past its last renewal instead of + * until someone intervenes, and that same span is the longest a second process has to wait before + * it may take the job over. */ @Injectable() export class CronLeaseService implements OnModuleInit { @@ -205,8 +219,8 @@ export class CronLeaseService implements OnModuleInit { * `reportContention` marks jobs for which losing the race is not a normal outcome. For a worker * job it is: the other worker holds the lease and is doing the work, and the result lands in the * database where everyone can see it. For a job whose effect is confined to the process that - * runs it, losing the race means that effect did not happen where it was needed — see - * PaymentCronService. Nothing else can tell the two apart, so the caller says which it is. + * runs it, losing the race means that effect did not happen where it was needed — which is what + * `CronScope.API` describes. Nothing else can tell the two apart, so the caller says which it is. */ async run(job: string, task: () => Promise, reportContention = false): Promise { // Once shutdown has begun, starting a run is worse than skipping it: `shutdown` waits on the @@ -278,10 +292,10 @@ export class CronLeaseService implements OnModuleInit { * lease is not worth activating that. * * A lease is NOT taken away from a job that is still working. Releasing on SIGTERM would hand - * over faster, but the job keeps running until the container's stop grace period ends it — - * `dfx-api-worker` is configured to allow two minutes — and a successor claiming the freed lease - * inside that window would run the same money-moving job alongside it. That is the outcome this - * mechanism exists to keep rare and short, so it is not traded for a faster handover. + * over faster, but the job keeps running until this process exits, which `main.ts` does once the + * wait below ends — so up to `SHUTDOWN_GRACE_MS` after the signal. A successor claiming the + * freed lease inside that window would run the same money-moving job alongside it, which is the + * outcome this mechanism exists to make rare, so it is not traded for a faster handover. * * What is still running after the wait therefore keeps its lease, which lapses within * `LEASE_TTL_SECONDS` of the last renewal. The renewal timers deliberately keep going meanwhile: diff --git a/src/shared/services/dfx-cron.service.ts b/src/shared/services/dfx-cron.service.ts index c53d0943e6..5ff7a49256 100644 --- a/src/shared/services/dfx-cron.service.ts +++ b/src/shared/services/dfx-cron.service.ts @@ -102,9 +102,12 @@ export class DfxCronService implements OnModuleInit { * or `lease unusable` is ALWAYS present, so a reader sees the current state rather than having * to count occurrences of a line that only appears when something is wrong — a count over a * window cannot tell "healthy" from "not reporting at all". Neither literal is a prefix of the - * other, and both sit at a fixed position, immediately after the job count. And the only free - * text — the reason — comes last, behind everything that is matched, because a free-text field - * BETWEEN matched fields can forge whatever field follows it. + * other. And the only free text — the reason — comes last, so every matched field sits at a + * fixed distance from the START of the line and no input can push one of them there. + * + * That last property is what a matcher has to be written to use: anchor at the start of the + * line, not at its end. The end of an unhealthy line is caller-supplied text, so a selector + * anchored there is deciding on a value the failure itself gets to choose. * * `__tests__/dfx-cron.service.spec.ts` pins both shapes; changing the wording here fails there. */ @@ -170,9 +173,10 @@ export class DfxCronService implements OnModuleInit { * configuration, and configuration can be wrong — a missed recreate leaves the old role in * place, `--scale` creates a second worker, a rollback puts two processes on `all`. In every * one of those the two processes hold separate locks and every payout runs twice, for as long as - * it takes someone to notice. The lease bounds that to its own expiry instead of reporting it a - * quarter of an hour later. It does not rule a double run out — CronLeaseService says under - * "What it does not do" exactly where it stops — so the jobs still have to tolerate a repeat. + * it takes someone to notice. With the lease, the second process has to take the claim before it + * may start the job, and while the holder keeps renewing it never gets one. It does not rule a + * double run out, and it does not bound how long one lasts — CronLeaseService says under "What + * it does not do" exactly where it stops — so the jobs still have to tolerate a repeat. * * `BOTH` jobs are deliberately exempt. They exist because a request path on THIS process reads * the state they maintain, so they have to run in every process — a lease over them would From 58daaff4a729922b5b18abada4e08ff3d407d905 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:26 +0200 Subject: [PATCH 55/86] Decide leaving Pending in the update, not in a status read processExpiredPayments runs in the worker while the expiry timers stay in the process that served the request, and cancelling and completing arrive from request paths, so several processes can read the same payment as Pending and each send the merchant its webhook and cancel the quotes again. A conditional update on the status decides who performs the transition; only the affected row triggers what follows. It holds for any number of processes and for every path into the transition. --- .../payment-link-payment.service.spec.ts | 109 +++++++++++++++++- .../services/payment-link-payment.service.ts | 53 ++++++++- 2 files changed, 155 insertions(+), 7 deletions(-) diff --git a/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts b/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts index fb4a45ab3a..e8ed980c5b 100644 --- a/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts +++ b/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts @@ -2,7 +2,8 @@ import { BlockchainRegistryService } from 'src/integration/blockchain/shared/ser import { FiatService } from 'src/shared/models/fiat/fiat.service'; import { In } from 'typeorm'; import { PaymentDevice, PaymentLinkPayment } from '../../entities/payment-link-payment.entity'; -import { PaymentLinkPaymentMode, PaymentLinkPaymentStatus } from '../../enums'; +import { PaymentQuote } from '../../entities/payment-quote.entity'; +import { PaymentLinkPaymentMode, PaymentLinkPaymentStatus, PaymentQuoteStatus } from '../../enums'; import { PaymentLinkPaymentRepository } from '../../repositories/payment-link-payment.repository'; import { PaymentActivationService } from '../payment-activation.service'; import { PaymentLinkPaymentService } from '../payment-link-payment.service'; @@ -63,6 +64,8 @@ describe('PaymentLinkPaymentService', () => { paymentLinkPaymentRepo = { find: jest.fn().mockResolvedValue([]), save: jest.fn().mockImplementation((entity) => entity), + // The default is the caller that wins the transition; the tests below that care set it. + update: jest.fn().mockResolvedValue({ affected: 1 }), } as unknown as jest.Mocked; paymentWebhookService = { sendWebhook: jest.fn() } as unknown as jest.Mocked; @@ -297,4 +300,108 @@ describe('PaymentLinkPaymentService', () => { expect(seen).toHaveLength(1); }); }); + + // --- Leaving Pending --- // + + /** + * Leaving `Pending` has to be decided by the database, not by a status read a moment earlier. + * The expiry job is a `Worker` job, the expiry timers stay in the process that served the + * request, and cancelling and completing arrive from request paths — so several processes can + * hold the same payment as `Pending` at the same time, and each of them would send the merchant + * its webhook and cancel the quotes again. + * + * `affected: 0` is what a caller that lost sees, and it is the whole assertion: nothing after + * the transition may happen for it. + */ + describe('leaving Pending', () => { + function transitions(): { status: PaymentLinkPaymentStatus }[] { + return paymentLinkPaymentRepo.update.mock.calls.map(([, values]) => values as { status: PaymentLinkPaymentStatus }); + } + + it('should expire through a conditional update rather than a status read', async () => { + await service.expirePayment(payment({ id: 7 })); + + expect(paymentLinkPaymentRepo.update).toHaveBeenCalledWith( + { id: 7, status: PaymentLinkPaymentStatus.PENDING }, + { status: PaymentLinkPaymentStatus.EXPIRED }, + ); + expect(transitions()).toHaveLength(1); + }); + + it('should not expire a second time when another process took the transition', async () => { + paymentLinkPaymentRepo.update.mockResolvedValue({ affected: 0 } as never); + + await service.expirePayment(payment({ id: 7, link: { webhookUrl: 'https://merchant.example/hook' } as never })); + + expect(paymentWebhookService.sendWebhook).not.toHaveBeenCalled(); + expect(paymentQuoteService.cancelAllForPayment).not.toHaveBeenCalled(); + expect(paymentActivationService.closeAllForPayment).not.toHaveBeenCalled(); + expect(paymentLinkPaymentRepo.save).not.toHaveBeenCalled(); + }); + + it('should cancel through the same transition', async () => { + await service.cancelByPayment(payment({ id: 7 })); + + expect(paymentLinkPaymentRepo.update).toHaveBeenCalledWith( + { id: 7, status: PaymentLinkPaymentStatus.PENDING }, + { status: PaymentLinkPaymentStatus.CANCELLED }, + ); + expect(paymentQuoteService.cancelAllForPayment).toHaveBeenCalledTimes(1); + }); + + it('should not cancel a payment the worker expired in between', async () => { + paymentLinkPaymentRepo.update.mockResolvedValue({ affected: 0 } as never); + + await service.cancelByPayment(payment({ id: 7, link: { webhookUrl: 'https://merchant.example/hook' } as never })); + + expect(paymentWebhookService.sendWebhook).not.toHaveBeenCalled(); + expect(paymentQuoteService.cancelAllForPayment).not.toHaveBeenCalled(); + expect(paymentLinkPaymentRepo.save).not.toHaveBeenCalled(); + }); + + /** The third way out of `Pending`, reached from a request path and from checkTxConfirmations. */ + describe('completing on a quote', () => { + function completing(values: Partial = {}): PaymentLinkPayment { + return payment({ + id: 7, + link: { configObj: { minCompletionStatus: PaymentQuoteStatus.TX_MEMPOOL } } as never, + ...values, + }); + } + + const quote = { id: 3, status: PaymentQuoteStatus.TX_MEMPOOL } as PaymentQuote; + + beforeEach(() => { + paymentQuoteService.getCompletedQuoteCount = jest.fn().mockResolvedValue(1); + paymentActivationService.closeAllForQuote = jest.fn(); + }); + + it('should complete a SINGLE payment through the transition', async () => { + await service['handleQuoteChange'](completing(), quote); + + expect(paymentLinkPaymentRepo.update).toHaveBeenCalledWith( + { id: 7, status: PaymentLinkPaymentStatus.PENDING }, + { status: PaymentLinkPaymentStatus.COMPLETED }, + ); + expect(paymentLinkPaymentRepo.save).toHaveBeenCalledTimes(1); + }); + + it('should not complete it again when another process got there first', async () => { + paymentLinkPaymentRepo.update.mockResolvedValue({ affected: 0 } as never); + + await service['handleQuoteChange'](completing(), quote); + + expect(paymentLinkPaymentRepo.save).not.toHaveBeenCalled(); + }); + + it('should count a MULTIPLE payment without taking a transition', async () => { + // It stays `Pending`, so there is nothing to claim — and claiming would keep every process + // but one from recording the quote it counted. + await service['handleQuoteChange'](completing({ mode: PaymentLinkPaymentMode.MULTIPLE }), quote); + + expect(paymentLinkPaymentRepo.update).not.toHaveBeenCalled(); + expect(paymentLinkPaymentRepo.save).toHaveBeenCalledTimes(1); + }); + }); + }); }); diff --git a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts index e99fc23ef1..1a76f42575 100644 --- a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts @@ -115,10 +115,41 @@ export class PaymentLinkPaymentService { } async expirePayment(payment: PaymentLinkPayment): Promise { + if (!(await this.takePendingTransition(payment, PaymentLinkPaymentStatus.EXPIRED))) return; + await this.doSave(payment.expire(), true); await this.cancelQuotesForPayment(payment); } + /** + * Moves a payment out of `Pending` in one statement, and answers whether THIS caller is the one + * that moved it. Everything that follows a transition — the merchant webhook, the quote + * cancellations, the activation closes — belongs to the caller that gets `true`. + * + * The read-then-write this replaces was safe while everything ran in one process. It is not any + * more: `processExpiredPayments` is a `Worker` job, while the expiry timers `createPayment` arms + * stay in the process that served the request, so two processes can read the same row as + * `Pending` and both act on it. A lock would not help, because they are different processes, and + * a lease would not either, because the request paths below reach the same transition without + * going through a job at all. + * + * The `status` in the criteria is what decides it: the database lets exactly one statement past + * `Pending` and reports it as the affected row, and every other caller gets nothing. That holds + * for any number of processes and for every path into the transition, which is why it sits here + * rather than at the call sites. + */ + private async takePendingTransition( + payment: PaymentLinkPayment, + status: PaymentLinkPaymentStatus, + ): Promise { + const { affected } = await this.paymentLinkPaymentRepo.update( + { id: payment.id, status: PaymentLinkPaymentStatus.PENDING }, + { status }, + ); + + return affected > 0; + } + async checkTxConfirmations(): Promise { const confirmingQuotes = await this.paymentQuoteService.getConfirmingQuotes(); @@ -425,11 +456,10 @@ export class PaymentLinkPaymentService { // expiry timers // // These stay in the process that served the request, although processExpiredPayments is a - // worker job. Moving the job did not give a payment a second timer: a timer is armed once, by - // the process that created the payment, and expirePaymentIfPending re-reads the payment with - // `status: Pending`, so a payment the job has already expired is left alone here. What the - // timers buy is what they bought before — a caller waiting on this process is released at the - // timeout rather than at the next tick of a job elsewhere. + // worker job. Both therefore race for the same payment, and neither the lock nor the lease + // spans them; what settles it is that the transition itself is atomic, see + // takePendingTransition. What the timers buy is what they bought before — a caller waiting on + // this process is released at the timeout rather than at the next tick of a job elsewhere. const scanTimeout = paymentLink.configObj.scanTimeout; if (scanTimeout) { setTimeout(() => this.expirePaymentIfPending(payment.id, true), scanTimeout * 1000); @@ -476,6 +506,10 @@ export class PaymentLinkPaymentService { } async cancelByPayment(payment: PaymentLinkPayment): Promise { + // Both callers reach here from a payment they read as `Pending`, and the worker can expire + // that same payment in between. Whoever the transition lets through sends the webhook. + if (!(await this.takePendingTransition(payment, PaymentLinkPaymentStatus.CANCELLED))) return; + await this.doSave(payment.cancel(), true); await this.cancelQuotesForPayment(payment); } @@ -610,7 +644,14 @@ export class PaymentLinkPaymentService { if (isPaymentComplete) { payment.txCount = await this.paymentQuoteService.getCompletedQuoteCount(payment, minCompletionStatus); - if (payment.mode === PaymentLinkPaymentMode.SINGLE) payment.complete(); + // The status read above is the same read-then-write as in expirePayment, and this one is + // reached from request paths as well as from checkTxConfirmations. A `MULTIPLE` payment + // stays `Pending` and has no transition to take: it only counts a quote. + if (payment.mode === PaymentLinkPaymentMode.SINGLE) { + if (!(await this.takePendingTransition(payment, PaymentLinkPaymentStatus.COMPLETED))) return; + + payment.complete(); + } await this.doSave(payment, true); } From d6cfefec0329a35ba66c32f50b9f638fa5ff9b09 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:27 +0200 Subject: [PATCH 56/86] Let the process flag switch off the dashboard start-up fill too The start-up call bypasses the scheduler, so it applied the role but not Process.LATEST_BALANCE_CACHE. Switching the job off still left one run per deployment. --- .../dashboard-financial.service.spec.ts | 21 +++++++++++++++++++ .../dashboard/dashboard-financial.service.ts | 12 +++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts b/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts index ef7063bf85..dfe80ec540 100644 --- a/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts +++ b/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts @@ -10,6 +10,7 @@ import { FinancialLogSummary } from '../../log/log.repository'; import { LogService } from '../../log/log.service'; import { DashboardFinancialService } from '../dashboard-financial.service'; import { LatestBalanceResponseDto } from '../dto/financial-log.dto'; +import * as ProcessService from 'src/shared/services/process.service'; import { Config, CronRole } from 'src/config/config'; import { TestUtil } from 'src/shared/utils/test.util'; import { LatestBalanceStore } from '../latest-balance.store'; @@ -26,6 +27,8 @@ describe('DashboardFinancialService', () => { afterEach(() => { if (originalRole !== undefined) Config.cronRole = originalRole; + + jest.restoreAllMocks(); }); beforeEach(async () => { @@ -46,6 +49,10 @@ describe('DashboardFinancialService', () => { service = module.get(DashboardFinancialService); originalRole ??= Config.cronRole; + + // DisabledProcess fails closed when the process flags were never loaded, which is the state of + // a bare testing module. The tests that care about the flag set it themselves. + jest.spyOn(ProcessService, 'DisabledProcess').mockReturnValue(false); }); function logEntry(): Log { @@ -543,6 +550,20 @@ describe('DashboardFinancialService', () => { expect(latestBalanceStore.set).not.toHaveBeenCalled(); }); + it('stays off at start-up when the process flag is off', async () => { + // The job carries Process.LATEST_BALANCE_CACHE, and this call bypasses the scheduler that + // applies it. Without the check, switching the job off still leaves one run per deployment. + Config.cronRole = CronRole.API; + jest.spyOn(ProcessService, 'DisabledProcess').mockReturnValue(true); + const getLatestFinancialLogSpy = jest.spyOn(logService, 'getLatestFinancialLog'); + + service.onModuleInit(); + await new Promise(process.nextTick); + + expect(getLatestFinancialLogSpy).not.toHaveBeenCalled(); + expect(latestBalanceStore.set).not.toHaveBeenCalled(); + }); + it('logs the failure of the start-up fill instead of swallowing it', async () => { // The scheduled run retries a minute later, so this must not throw - but a silent failure // would leave the endpoint empty with no trace of why. diff --git a/src/subdomains/supporting/dashboard/dashboard-financial.service.ts b/src/subdomains/supporting/dashboard/dashboard-financial.service.ts index dd8787f55b..742b97f7eb 100644 --- a/src/subdomains/supporting/dashboard/dashboard-financial.service.ts +++ b/src/subdomains/supporting/dashboard/dashboard-financial.service.ts @@ -4,7 +4,7 @@ import { Config, CronRole } from 'src/config/config'; import { Asset } from 'src/shared/models/asset/asset.entity'; import { AssetService } from 'src/shared/models/asset/asset.service'; import { DfxLogger } from 'src/shared/services/dfx-logger'; -import { Process } from 'src/shared/services/process.service'; +import { DisabledProcess, Process } from 'src/shared/services/process.service'; import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { RefRewardService } from '../../core/referral/reward/services/ref-reward.service'; import { AssetLog, BalancesByFinancialType, FinanceLog } from '../log/dto/log.dto'; @@ -36,9 +36,17 @@ export class DashboardFinancialService implements OnModuleInit { onModuleInit() { // Fills the store once at start-up instead of leaving it empty until the first scheduled run: // getLatestBalance answers from the store and nothing else, so until the first fill the - // endpoint has no value to return. + // endpoint has no value to return. The two conditions below are the ones the scheduler applies + // to the job itself, and this call bypasses the scheduler entirely. + // + // The role first: the job is scoped `api` because a request path is the only reader of the + // store. In the worker the aggregation would be spent on a value no request there can read. if (Config.cronRole === CronRole.WORKER) return; + // Then the flag: a job switched off through DISABLED_PROCESSES has to stay off, including at + // start-up. Otherwise switching it off still leaves one run per deployment. + if (DisabledProcess(Process.LATEST_BALANCE_CACHE)) return; + void this.refreshLatestBalance().catch((e) => // Not rethrown: a failed first fill leaves the store empty, which the endpoint already // handles, and the scheduled run retries a minute later. Swallowing it silently would hide From 7b3aeac0b3ee85f6cc98cfc14516bb5aa14baf53 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:28 +0200 Subject: [PATCH 57/86] Correct the sweep order and four comment claims The connection sweep drops its entry before terminating the socket, so a terminate that throws cannot leave the entry behind and stall every following sweep on it. The comments: the payment-link domain has three Worker jobs, not one; the metric export interval is only read when an OTLP endpoint is configured; and the scope on ExchangeController::checkTrades applies to a job that is not registered anywhere, because the class is a controller. --- .env.example | 3 ++- .../controllers/exchange.controller.ts | 4 ++++ .../__tests__/payment-link.gateway.spec.ts | 22 +++++++++++++++++++ .../controllers/payment-link.gateway.ts | 6 ++++- .../services/payment-link-fee.service.ts | 7 +++--- 5 files changed, 37 insertions(+), 5 deletions(-) diff --git a/.env.example b/.env.example index dcc083f0cc..1a7bcb5de0 100644 --- a/.env.example +++ b/.env.example @@ -294,7 +294,8 @@ S3_ADMIN_SECRET_KEY= OTEL_EXPORTER_OTLP_ENDPOINT= # Metric export interval in milliseconds. Optional: unset leaves the SDK default (60s). # A shorter interval costs a full collect-and-export of every instrument on the event loop this -# split is meant to keep free. An invalid or non-positive value aborts the boot. +# split is meant to keep free. It is only read when OTEL_EXPORTER_OTLP_ENDPOINT is set; with an +# endpoint, an invalid or non-positive value aborts the boot, and without one it is never looked at. # OTEL_METRIC_EXPORT_INTERVAL=60000 FIXER_BASE_URL= diff --git a/src/integration/exchange/controllers/exchange.controller.ts b/src/integration/exchange/controllers/exchange.controller.ts index a8d766b2c3..35dd9d4044 100644 --- a/src/integration/exchange/controllers/exchange.controller.ts +++ b/src/integration/exchange/controllers/exchange.controller.ts @@ -174,6 +174,10 @@ export class ExchangeController { // Api, not Both: `trades` is filled by POST :exchange/trade and read by GET trade/:id, the // request paths of this controller shown below. In a process those requests never reach, the // map stays empty and this job has nothing to work on. + // + // Nothing acts on that choice today. DfxCronService reads the decorator off providers, and this + // class is registered under `controllers` in ExchangeModule, so the job is not registered in any + // process. The scope says where it would belong if it ever were, not where it runs. @DfxCron(CronExpression.EVERY_30_SECONDS, { scope: CronScope.API, timeout: 1800 }) async checkTrades() { const openTrades = Object.values(this.trades).filter(({ status }) => status === TradeStatus.OPEN); diff --git a/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts b/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts index 09e4d380e2..070535f26b 100644 --- a/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts +++ b/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts @@ -174,6 +174,28 @@ describe('PaymentLinkGateway', () => { expect(deviceIds()).toEqual(['pos-1']); }); + it('drops the entry even when terminating the socket throws', () => { + // A socket implementation that throws on terminate used to leave its entry behind, and the + // exception left both loops. The same entry then threw first on every following sweep, so + // the map it exists to bound never shrank again and no other device was ever checked. + const failing = connect('pos-1'); + failing.terminate = () => { + throw new Error('socket already destroyed'); + }; + const other = connect('pos-2'); + + gateway.checkConnections(); + + expect(() => gateway.checkConnections()).toThrow('socket already destroyed'); + expect(deviceIds()).toEqual(['pos-2']); + + // The sweep gets past it from here on, which is what makes the failure single-shot. + gateway.checkConnections(); + + expect(other.terminated).toBe(true); + expect(deviceIds()).toEqual([]); + }); + it('runs in every process, because every process holds its own sockets', () => { const params: DfxCronParams = Reflect.getMetadata( DFX_CRONJOB_PARAMS, diff --git a/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts b/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts index 7fd4c0ae7d..da10e21aaa 100644 --- a/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts +++ b/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts @@ -82,8 +82,12 @@ export class PaymentLinkGateway implements OnGatewayConnection, OnModuleInit { for (const [device, connections] of this.clients) { for (const [clientId, connection] of connections) { if (!connection.responsive) { - connection.socket.terminate(); + // Dropped from the map BEFORE the socket is terminated. The other way round, a + // `terminate` that throws left the entry in place and took the exception out of both + // loops, so every device after this one went unchecked — and the same entry threw first + // on the next sweep, and on every one after that. this.removeClient(device, clientId); + connection.socket.terminate(); continue; } diff --git a/src/subdomains/core/payment-link/services/payment-link-fee.service.ts b/src/subdomains/core/payment-link/services/payment-link-fee.service.ts index 91a43c9e40..e6061f4aee 100644 --- a/src/subdomains/core/payment-link/services/payment-link-fee.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-fee.service.ts @@ -42,9 +42,10 @@ export class PaymentLinkFeeService implements OnModuleInit { * Scope Api, not Both: the cache it fills is a field of this service, and the only reader is * getMinFee below. Following that chain out — createTransferAmount -> createTransferAmounts -> * createQuote / createPayRequest, plus the Binance webhook handler — every caller is a request - * path or one of the Api-scoped payment crons. The one Worker job in this domain, - * PaymentCronService::forwardDeposits, takes its fee rate from BitcoinFeeService and never - * reaches this cache. + * path or one of the Api-scoped payment crons. None of the Worker jobs in this domain reaches + * the cache: PaymentCronService::forwardDeposits takes its fee rate from BitcoinFeeService, and + * ::processExpiredPayments and ::checkTxConfirmations price nothing — they move a payment out of + * `Pending` and cancel or close what hangs off it. * * Both would also break the rule this scope mechanism introduced: a job that runs in every * process must be harmless twice over, and this one queries gas prices for eight EVM chains From 97b774d3587cb4f87c0d8ed55e002c9d4f86d814 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:29 +0200 Subject: [PATCH 58/86] Apply Prettier formatting --- .../services/__tests__/cron-registration.guard.spec.ts | 4 +++- .../services/__tests__/payment-link-payment.service.spec.ts | 4 +++- .../payment-link/services/payment-link-payment.service.ts | 5 +---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/shared/services/__tests__/cron-registration.guard.spec.ts b/src/shared/services/__tests__/cron-registration.guard.spec.ts index 8f19d19ce3..c0a52dcabe 100644 --- a/src/shared/services/__tests__/cron-registration.guard.spec.ts +++ b/src/shared/services/__tests__/cron-registration.guard.spec.ts @@ -83,7 +83,9 @@ describe('cron registration', () => { // pattern that matches nothing at all looks like. This runs the same filter over a file that // does contain each pattern, so a check that can no longer find anything fails here. for (const { pattern } of FORBIDDEN) { - const planted = [{ path: 'planted.ts', content: `class X { @Cron() @Interval() @Timeout() f() { setInterval(); } }` }]; + const planted = [ + { path: 'planted.ts', content: `class X { @Cron() @Interval() @Timeout() f() { setInterval(); } }` }, + ]; expect(planted.filter((f) => pattern.test(f.content)).map((f) => f.path)).toEqual(['planted.ts']); } diff --git a/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts b/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts index e8ed980c5b..919ef85d58 100644 --- a/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts +++ b/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts @@ -315,7 +315,9 @@ describe('PaymentLinkPaymentService', () => { */ describe('leaving Pending', () => { function transitions(): { status: PaymentLinkPaymentStatus }[] { - return paymentLinkPaymentRepo.update.mock.calls.map(([, values]) => values as { status: PaymentLinkPaymentStatus }); + return paymentLinkPaymentRepo.update.mock.calls.map( + ([, values]) => values as { status: PaymentLinkPaymentStatus }, + ); } it('should expire through a conditional update rather than a status read', async () => { diff --git a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts index 1a76f42575..a2abd94281 100644 --- a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts @@ -138,10 +138,7 @@ export class PaymentLinkPaymentService { * for any number of processes and for every path into the transition, which is why it sits here * rather than at the call sites. */ - private async takePendingTransition( - payment: PaymentLinkPayment, - status: PaymentLinkPaymentStatus, - ): Promise { + private async takePendingTransition(payment: PaymentLinkPayment, status: PaymentLinkPaymentStatus): Promise { const { affected } = await this.paymentLinkPaymentRepo.update( { id: payment.id, status: PaymentLinkPaymentStatus.PENDING }, { status }, From eac6159a8a9e6511dd09bac34e710af456e91158 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:31 +0200 Subject: [PATCH 59/86] Say what the renewal margin actually covers --- src/shared/services/cron-lease.service.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/shared/services/cron-lease.service.ts b/src/shared/services/cron-lease.service.ts index 0f6460e4bb..5bdf2b7aef 100644 --- a/src/shared/services/cron-lease.service.ts +++ b/src/shared/services/cron-lease.service.ts @@ -349,8 +349,12 @@ export class CronLeaseService implements OnModuleInit { * answers slowly is exactly the situation this has to survive: the attempts pile up, each one * occupying a pooled connection, and an older answer can land after a newer one. Re-arming only * once the previous attempt has settled bounds that to a single outstanding statement. The price - * is that the renewals drift apart by however long the database takes to answer, which the lease - * TTL — three times the interval — is sized to absorb. + * is that the renewals drift later by however long the database takes to answer, and the TTL — + * three times the interval — leaves room for one such answer to be slow or lost, not for a + * database that is slow to every one of them. See RENEWAL_INTERVAL_MS. + * + * Losing the claim does not stop the run. There is nothing here that could stop it, and the + * timer deliberately keeps going: this process holds the claim for as long as it can renew it. */ private keepAlive(job: string): { stop: () => void } { let stopped = false; From 591fa4a933f553fea5ed5c93ae2d8ac0b89eadef Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:32 +0200 Subject: [PATCH 60/86] Carry the effects of leaving Pending through the transition The conditional update committed on its own, so a caller that stopped between it and the quote cancellations left a payment out of Pending with its quotes still open. processExpiredPayments asks for Pending and would never see that row again. The effects that write to the database now run in the transaction that takes the transition, and the merchant webhook stays outside it. --- PR-BODY-4537.md | 349 ++++++++++ .../payment-link-payment.service.spec.ts | 164 ++++- .../services/payment-activation.service.ts | 625 +++++++++--------- .../services/payment-link-payment.service.ts | 78 ++- .../services/payment-quote.service.ts | 15 +- 5 files changed, 868 insertions(+), 363 deletions(-) create mode 100644 PR-BODY-4537.md diff --git a/PR-BODY-4537.md b/PR-BODY-4537.md new file mode 100644 index 0000000000..ece8ec3fc1 --- /dev/null +++ b/PR-BODY-4537.md @@ -0,0 +1,349 @@ +> [!IMPORTANT] +> **`CRON_ROLE` must be set in every environment before this is deployed.** The boot aborts +> without it — deliberately, see below — so an environment missing the variable fails on the next +> deploy, whatever that deploy was for. The two repositories have separate pipelines with no +> dependency between them: merging here triggers a deploy without checking whether the variable is +> already in place. The configuration change is a separate, already open PR in the infrastructure +> repository; it is inert for the currently running image. + +> [!WARNING] +> **This PR bundles four subsystems, and a revert is all-or-nothing across all four.** That is a +> deliberate decision, recorded here so it does not surprise anyone at merge time. See +> [What this PR bundles](#what-this-pr-bundles) directly below before approving. + +## What this PR bundles + +The title says process separation, and that is the reason the branch exists. It is not all the +branch contains. Four independently revertable subsystems ended up on one branch, and because they +are one branch they are also one revert: pulling any of them out after the fact pulls out the other +three. + +| Area | What is in here | Why it is entangled | +|---|---|---| +| **Process split** | `CRON_ROLE`, `CronScope`, mandatory `scope` on 140 `@DfxCron` declarations, boot log, role heartbeat | The subject of the PR | +| **Cross-process lease** | `cron_lease` table, migration, entity, `CronLeaseService`, shutdown handler in `main.ts` | Only meaningful once two processes exist, and the split is what makes the in-process lock insufficient | +| **OpenTelemetry metrics pipeline** | `src/runtime-metrics.ts`, new dependencies, new `OTEL_METRIC_EXPORT_INTERVAL` | Written to measure the event loop saturation that motivated the split; independent of it at runtime | +| **`MonitoringService` rework** | DB-backed read path, 30 s process cache, `pessimistic_write` merge on write | Forced by the split: with the observers on the worker, the API process has no state of its own to serve | +| **`DashboardFinancialService` rework** | Aggregation stays in the worker, response building becomes its own `api` job, `LatestBalanceStore` | Forced by the split, for the same reason | + +Two of the five (metrics pipeline, and the lease insofar as it changes the schema) would stand on +their own. They are not being split out: the operator has decided against a follow-up PR, so the +bundling is stated instead of removed. What that means concretely: + +- **A revert of this PR removes the OTLP event-loop metrics** and any dashboard or alert built on + them, whether or not the reason for reverting had anything to do with them. +- **A revert re-introduces the monitoring read path that answers from process memory**, which is + wrong in a two-process deployment — so a revert here has to be accompanied by a rollback of the + role configuration, not just of this code. +- **A revert does not drop the `cron_lease` table.** The migration is not reverted by reverting the + code; the table is simply left unused. + +The safe rollback is not a revert of this PR but `CRON_ROLE=all`, under which every process +registers every job. That is the mode this branch is designed to be merged in. + +## Why + +Background jobs and HTTP requests share a single Node event loop, and Node runs JavaScript on one +thread. A CPU-heavy job therefore delays every request in the same process. + +Measured in production while investigating slow API responses: + +| Signal | Value | +|---|---| +| Event loop utilization | 84% mean, p90 100% | +| Event loop delay | mean 339 ms, p95 up to 5.3 s | +| `/version` (no DB access, 1 ms handler), measured locally against the container | p50 7 ms, **p95 5.5 s**, max 16 s | +| Actual HTTP load | 1.9 req/s | + +A p50 of a few milliseconds next to a p99 in the seconds is the signature of a blocked loop, not of +slow work: the same distribution reproduces on a route touching neither the database nor any +external service, and it persists when the network path is bypassed entirely. Database, host and +network were each ruled out separately — during one 6.1 s freeze the database had zero active +queries, and the connection pool never had a single waiting request. + +The cause is the scheduler, not traffic. Grouping the delay measurements by second-within-minute +over six hours (2,154 windows) gives stall rates of 5.8 / 52.8 / 73.3 / 81.8 / 50.0 / 21.2 % — a +factor of 14, following the scatter curve of the job start delay exactly. Request traffic in the +same grid varies by a factor of 2 and peaks elsewhere. + +## What changes + +This PR is the application half of running the same image twice: one process serving HTTP, one +running the background work. Under `CRON_ROLE=all` no existing `@DfxCron` is left out on account of +its scope — new are the role heartbeat, the Spark wallet maintenance job, `refreshLatestBalance`, +the payment delivery job, the WebSocket liveness sweep and the registration logs. + +**1. `CRON_ROLE` decides which jobs a process registers**, and `CronScope` says which process a job +belongs to. Three values each, because two would not be enough in either direction: + +- A role `all` is needed because otherwise no value covers every job — local development, the test + suite and any deployment without a separate worker would each lose something. +- A scope `api` is needed because some work is bound to the process holding the open connections — + state a request path reads in *this* process, which a job running elsewhere cannot maintain. + Delivering to those connections is a different matter and is no longer done from an `api`-scoped + job; see [The api scope and the lease](#the-api-scope-and-the-lease-pull-against-each-other) for + why that separation had to be made. + +The variable has no default and an unknown or empty value aborts the boot. Every possible default +is silent in one direction: `worker` would make a misconfigured API process run all background work +twice, `api` would make a misconfigured worker do nothing at all. + +**2. `scope` is mandatory on every `@DfxCron`.** A wrong classification fails silently — a job +wrongly scoped `worker` leaves the cache it maintains empty in the process that reads it, with no +error anywhere. A default plus an exception list moves that decision into a hand-maintained list +the compiler never sees; such a list grows and goes stale, and pinning it in a test proves the +state of the list rather than the property it stands for. + +Counted from the source tree rather than carried forward: **140 `@DfxCron` declarations — 119 +`worker`, 5 `api`, 16 `both`**, across 98 files and 34 areas; 118 carry a `process` flag, 22 do +not, five of those deliberately. 139 of the 140 have a registration path. `docs/cron-jobs.md` +carries the assignment per job, generated from the decorators, and both of its distribution tables +sum to 140. + +**3. Cases where a scope alone would not have been enough:** + +- **Monitoring state** (11 observers behind `GET /health*` and `/monitoring/data`). Scoping the + observers to the worker would freeze eleven endpoints at the boot snapshot in the API process, + the health report included; scoping them to the API process would move AML, node and bank queries + back into the request path. The state is already persisted in full, so the read path takes it from + there with a 30-second process cache. The write path needed its own answer: every process writes + the whole state as a single row, so only the metrics changed in this process are merged into the + stored row — otherwise whichever writes last drops the other's work. +- **The dashboard balance store**, written by the financial aggregation and read without touching + the database. That property was measured (23 ms median, 1,989 ms p95 before it existed) and is + kept: building the response becomes its own minute job scoped `api`, and the aggregation stays in + the worker. +- **Periodic work registered outside the scheduler** — two native `@Cron` decorators and a bare + `setInterval` driving on-chain wallet maintenance. All three would have run in both processes. + A test forbids the patterns, and carries no exceptions. + +**4. Four jobs gain a `process` flag** so a single misbehaving job can be stopped at runtime +instead of by deploy, and the worker reports itself under its own service name so its outgoing +calls do not read as API traffic. + +## Cross-process lease + +An architecture review named the load-bearing assumption of the original design: *"there is exactly +one worker, and the configuration is right"*. It was held up by convention, a runbook sentence and +an alert that **reports** a double run 15–25 minutes later. For a path that moves money, detection +is the second-best answer — and `LockClass` keeps its state in a field in process memory, so it +cannot see a second process at all. + +**Jobs scoped `worker` or `api` hold a lease in the `cron_lease` table for the duration of their +run.** A single statement decides it: the upsert takes the row over only if it has expired. Two +processes racing are serialised by the primary key and one of them gets a row back. + +**What that is worth, stated precisely.** The lease does not exclude a double run, and it does +not bound how long one lasts. A job runs once because the deployment runs one worker and because +the job tolerates being run again. What the lease adds underneath those two properties is that a +second process has to take the claim before it may **start** the job: while the holder keeps +renewing, a wrongly configured second process — a missed recreate, a second worker from `--scale`, +two processes left on `all` after a rollback — does not start it at all. If the holder stops +renewing while it is still working, the claim lapses, the second process starts, and the first runs +on to its own end; nothing here shortens that, because a running function cannot be aborted from +the outside in JavaScript and a cooperative check at every write is the same work as the fencing +token this does not carry. `CronLeaseService` says so in its own "What it does not do". + +What the lease **does** bound is the waiting: a claim left behind by a process that was killed +blocks the job for the lease TTL rather than until somebody reads an alert, and that same span is +the longest a second process waits before it may take the job over. + +**Scope `both` is deliberately exempt.** Those jobs maintain state a request path reads in *their +own* process; a lease over them would starve whichever lost the race and freeze that state. Their +safety comes from a different property: running twice has to be harmless by construction, which is +what CONTRIBUTING requires of them. + +**Why a lease and not an advisory lock.** `pg_advisory_lock` is bound to the connection and would +hold a pooled connection for the whole runtime of the job. 67 jobs declare a timeout measured in +minutes; that is a real risk to a connection pool sized by `SQL_POOL_MAX`. An expiring row +costs one short query each to take, extend and release. + +**The lease is 60 seconds, renewed every 20, and unrelated to the job's timeout.** The expiry bounds +how long a claim outlives an owner that can no longer speak for itself — SIGKILL, an OOM kill, a +lost machine. That is a property of the failure mode, not of the work, and deriving it from the +job's own timeout got it backwards: `timeout` is measured in seconds and nineteen `@DfxCron` +declarations carry 7200, so a process killed mid-run used to block its own successor from that job +for up to two hours, silently. + +**Shutdown is the other half.** Nothing in this repository ever asked for a shutdown hook, so +SIGTERM ended the process instantly and the release in the `finally` never ran. +`CronLeaseService.shutdown` now waits up to ten seconds for the runs this process holds, so their +normal release hands the job to the successor. A run still going after that **keeps** its lease: +taking it away would let the successor start the same job while this process works on it for the +rest of the grace, which is the outcome the lease exists to make rare. + +It hangs off a SIGTERM/SIGINT handler rather than `app.enableShutdownHooks()`, and that is +deliberate. The idiomatic call is global: it would also start running the nine `onModuleDestroy` +implementations this application carries but has never executed, and Nest runs those *before* the +lease hook — they empty the strategy registries that PayIn, PayOut and DEX jobs resolve from. On +its own that is harmless, since the process was about to die. Next to this change it is not: the +wait deliberately keeps in-flight jobs alive longer, so a running payout would gain time to fail on +an emptied registry rather than simply be cut off. A test pins that the idiomatic call stays out. + +**An unusable lease table is reported rather than silent.** Without it every worker- and api-scoped +job is skipped on every tick — the right behaviour, and what CONTRIBUTING asks for, but the skip +used to look exactly like a job with nothing to do. The role heartbeat is scope `both` and +therefore exempt from the lease, so it kept reporting a healthy process while everything it counts +sat out; and it counts *registered* jobs, which cannot see this at all. The lease layer now reads +the table at start-up and carries a health flag that stays false until an operation gets through; +the heartbeat writes the same line at error level with the reason appended when it is bad. The same +line, because the role alert matches on its shape. + +**What it still cannot do — and the code says so.** If the holder stops renewing while a job runs, +the claim expires and a second process can start the job while the first is still working, for as +long as that first run takes. Full fencing would need a token on every single write. What the lease +shortens is the wait, not the overlap. + +**With the database unreachable the job does not run.** A job that moves money must not proceed on +the assumption that it is probably alone — and that is exactly when the assumption is least safe. + +### The api scope and the lease pull against each other + +`PaymentCronService` writes to the database and triggers merchant webhooks, which argues for one +process, and it used to be the only thing releasing the callers waiting on the process that ran it, +which argues for every process. A single scope cannot satisfy both, and the lease made that +visible: whichever process lost the claim left its callers waiting for nothing. + +The resolution taken is to split the job rather than the scope. The writing runs under the lease +(`Worker`), and `deliverPaymentUpdates` delivers from the state those writes leave behind, scoped +`Both`, in every process, without a lease — it writes nothing and calls nothing outside its own +process, which is what allows it to run everywhere. An `api`-scoped job losing the race is still +logged at error level, unlike a worker job, which loses it every cycle by design. + +Leaving `Pending` is decided by the update statement rather than by a status read, because the +expiry timers stay in the process that served the request while the expiry job runs in the worker, +and cancelling and completing arrive from request paths as well. Only the caller the database lets +past `Pending` sends the merchant its webhook. + +## Schema + +`migration/1785600000000-AddCronLease.js` creates `cron_lease`. Two things worth stating: + +- The primary key is `PK_a12c181c2b26f33be13d55a15af` — `PK_` plus the first 27 characters of + `sha1('cron_lease_name')`, which is what TypeORM's own naming strategy produces. A hand-picked + name would not be recognised by a schema comparison, which would then offer to create the + constraint. A new test recomputes every primary key declared in a `CREATE TABLE` across all + migrations (102 of them, all matching) and rejects any spelled-out constraint name. +- `acquired` and `expires` are `timestamptz`. They are compared against `now()` in raw SQL, and a + value without a zone on one side of that comparison resolves through whatever time zone the + session carries: the same row expires an hour late or an hour early across a daylight saving + change. An hour late is a job that runs nowhere, an hour early is two processes running it. + +`src/shared/models/cron-lease/cron-lease.entity.ts` mirrors the table. The service never reads +through a repository — the claim is a single `INSERT .. ON CONFLICT .. WHERE` the query builder +cannot express — but a table that exists only as DDL is invisible to the entity model, and the next +generated migration would read that absence as an instruction to `DROP TABLE "cron_lease"`. A test +builds the entity metadata without a connection and checks it against the migration file. + +## Deployment + +The order matters and is not optional: + +1. **`CRON_ROLE=all` must be set in every environment before this PR is deployed.** The boot aborts + without it, so an environment missing the variable fails on the next routine deploy — for the + currently running image the variable is unknown and therefore inert. +2. Merge and deploy this PR. The migration creating `cron_lease` runs with it. +3. Observe: the boot log states the split — `CronRole all: registered 139 of 139 jobs (worker: 119, + api: 4, both: 16)` — and health and dashboard endpoints answer unchanged. +4. Alerting, log-level normalisation, runbooks and dashboards — before, not after the next step. +5. Create the second process and set the roles. With the roles split, the boot log reads + `registered 135 of 139` in the worker and `registered 20 of 139` in the HTTP process. + +Rolling back never means reverting this PR: under `CRON_ROLE=all` its content is today's behaviour, +so what gets reset is configuration. See [What this PR bundles](#what-this-pr-bundles) for why a +revert is the expensive option. + +## Testing + +`npm test`, plus targeted runs on the affected suites; `tsc --noEmit`, ESLint and Prettier clean. +Coverage added by this branch: + +- every rejected value for the role, including the empty string and the absent variable, asserting + a throw rather than a silent default +- the `env -> Config` wiring the cron service actually reads, not just the parser +- registration per role: `all` registers everything, each role drops the other's scope, `both` + survives in all three +- which jobs pass through the lease, and that a job declaring `timeout: 7200` still claims for 60 + seconds — the regression that made a deployment block a job for two hours +- shutdown: the lease survives a shutdown that outlasts the grace period, the shutdown does not + return before the job does, `main.ts` is pinned to wire it to the signal at all, and pinned NOT + to reach for `app.enableShutdownHooks()` +- an unusable lease table: reported at start-up, still reported on the next heartbeat when no new + failure occurred, and reported healthy again once a claim gets through +- an api-scoped job losing the race is reported; a worker job losing it is not +- constraint naming across every migration, and the `cron_lease` entity against its own DDL +- the monitoring service: read path answers from the persisted state including the filtered + queries, the merge prefers whichever value is newer, the write path keeps metrics another process + wrote, the row is read under a write lock in the same transaction it writes in, an older + measurement is not put back over a newer one, and the merge is retried only on errors a retry can + resolve +- the monitoring state row: an environment whose state does not live under `id: 1` is answered from + the row that exists, and the write seeds `id: 1` from it rather than with a partial state +- the statistic start-up fill: not in the worker, yes in `api` and `all`, off when the process flag + is off, and a failure reported instead of left as an unhandled rejection +- the Spark wallet maintenance: registered as a job rather than a timer, scoped `worker` +- the guard against `@Cron(`, `@Interval(`, `@Timeout(` and `setInterval(`, including a check that + its exception list still matches something + +Every fix in the review rounds below was verified by putting the defect back and watching the test +fail, then restoring it. + +## Known discrepancies, recorded rather than fixed + +`ExchangeController::checkTrades` is **never registered**: its class is listed under `controllers:` +in `ExchangeModule` and nowhere under `providers:`, and `DiscoveryService.getProviders()` does not +return controllers, so the scan never sees the decorator. This predates the process split — the job +has never run. `TransactionController::checkLists` looks like the same case but is not: +`HistoryModule` lists that class under both `controllers:` and `providers:`, so the job is +registered, on the provider instance, which is a different object from the controller instance the +request handlers use. + +`CitreaBaseStrategy::checkPayInEntries` is declared on an **abstract** class, so it is registered +once per concrete subclass rather than once per declaration. There is currently one subclass. + +16 jobs carry no `process` flag. That is pre-existing, named in `docs/cron-jobs.md` as an omission +rather than hidden, and retrofitting it means introducing 16 new kill switches — a decision about +those jobs, not about this PR. + +What should happen to any of these is a decision about the jobs, not about this inventory. + +## Review rounds + +**Round 1 — rebased** onto the current `develop`, conflict-free, CI 12/12 green. + +**Round 2 — role heartbeat added.** `DfxCronService::reportRole` writes +`CronRole : heartbeat, N jobs registered` in every process every ten minutes. The reason lies +outside this repository but the line originates here: the Grafana rule meant to report a wrong role +assignment used to read the boot line, which is written exactly once. On a healthy system a +counting window over it reports permanently from the day after the last deploy, because the line +falls out of the window while the container keeps running. And the most expensive state is +precisely the one without a restart: if the recreate for a configuration change does not happen, +the HTTP process keeps its old role while the worker takes over the same jobs. Without a restart +there is no new boot line, so the rule could not see it structurally. Scope `both` so the line +appears in every process, with the role *in* the line; no `process` flag, because a watchdog that +can be switched off looks, switched off, exactly like the failure it reports. `useDelay: false`, +because the alert reads a 12-minute window and the jitter is adjustable from outside through +`CRON_JOB_DELAY` — a watchdog must not have its timing tuned by a knob meant for spreading load. + +**Round 3 — seven review points**, three implemented, four re-measured and declined with the +measurement stated. Implemented: the heartbeat tests were reading decorator metadata and would have +stayed green if the scan never saw the method, so the test now hands the scan its own service +instance the way Nest does and expects the heartbeat in the count; the guard test's `setTimeout` +gap was documented with its evidence; and the claim "behaviourally identical to today under +`CRON_ROLE=all`" was narrowed to what is actually true. Declined: the `git diff --check` whitespace +report (all files are CRLF and `.prettierrc` sets `endOfLine: auto`; two consecutive `develop` +commits report the same), the `PaymentCronService` scope (see above — the concern was right and is +now addressed by the lease and by reporting the lost race), the 16 missing flags, and the +"environment updates missing" point (they are in the infrastructure repository, because that is +where `CRON_ROLE` is set). + +**Round 4 — the cross-process lease**, described above. + +**Round 5 — a five-instance review of this PR and its three infrastructure counterparts.** Nine +findings here, all fixed on this branch: the lease TTL derived from the job timeout (up to a +two-hour outage per deployment) and the missing shutdown path; an unusable lease table looking like +a healthy process; a primary key name that violates the deterministic-naming rule; the missing +entity; timestamps without a time zone; the Spark maintenance timer and the statistic start-up fill +both running outside the scheduler and therefore outside the lease; the `api` scope contradicting +the lease; the monitoring read path depending on a row with `id: 1`; and this bundling, which is +named here rather than split out. diff --git a/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts b/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts index 919ef85d58..f580408733 100644 --- a/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts +++ b/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts @@ -1,6 +1,7 @@ +import { ConfigService, GetConfig } from 'src/config/config'; import { BlockchainRegistryService } from 'src/integration/blockchain/shared/services/blockchain-registry.service'; import { FiatService } from 'src/shared/models/fiat/fiat.service'; -import { In } from 'typeorm'; +import { EntityManager, In } from 'typeorm'; import { PaymentDevice, PaymentLinkPayment } from '../../entities/payment-link-payment.entity'; import { PaymentQuote } from '../../entities/payment-quote.entity'; import { PaymentLinkPaymentMode, PaymentLinkPaymentStatus, PaymentQuoteStatus } from '../../enums'; @@ -60,12 +61,57 @@ describe('PaymentLinkPaymentService', () => { sockets.set(deviceId, since); } + /** + * The row the transition competes for, and the only thing that says whether a caller won: the + * manager below applies an update exactly when its criteria still match, as the database does. + */ + let row: PaymentLinkPayment; + + /** + * Stands in for the transaction the transition runs in, statements included. Statements are + * staged and written back to `row` only when the callback returns — a callback that throws + * leaves the row as it was, which is what a rollback means and what these tests are about. + */ + function transaction(run: (manager: EntityManager) => Promise): Promise { + const staged = Object.assign(new PaymentLinkPayment(), row); + + return run(transactionManager(staged)).then((result) => { + Object.assign(row, staged); + + return result; + }); + } + + function transactionManager(staged: PaymentLinkPayment): EntityManager { + const update = jest.fn().mockImplementation((_target, criteria: number | Partial, values) => { + const matches = typeof criteria === 'number' || criteria.status == null || criteria.status === staged.status; + if (matches) Object.assign(staged, values); + + return Promise.resolve({ affected: matches ? 1 : 0 }); + }); + + managerUpdates.push(update); + + return { update } as unknown as EntityManager; + } + + /** Every statement any transition ran, in order, as [criteria, values]. */ + function transitions(): [Partial, Partial][] { + return managerUpdates.flatMap((update) => + update.mock.calls.map(([, criteria, values]) => [criteria, values] as [never, never]), + ); + } + + let managerUpdates: jest.Mock[]; + beforeEach(() => { + row = payment({ id: 7 }); + managerUpdates = []; + paymentLinkPaymentRepo = { find: jest.fn().mockResolvedValue([]), save: jest.fn().mockImplementation((entity) => entity), - // The default is the caller that wins the transition; the tests below that care set it. - update: jest.fn().mockResolvedValue({ affected: 1 }), + manager: { transaction: jest.fn().mockImplementation(transaction) }, } as unknown as jest.Mocked; paymentWebhookService = { sendWebhook: jest.fn() } as unknown as jest.Mocked; @@ -310,28 +356,20 @@ describe('PaymentLinkPaymentService', () => { * hold the same payment as `Pending` at the same time, and each of them would send the merchant * its webhook and cancel the quotes again. * - * `affected: 0` is what a caller that lost sees, and it is the whole assertion: nothing after - * the transition may happen for it. + * A row that no longer reads `Pending` is what a caller that lost meets, and it is the whole + * assertion: nothing after the transition may happen for it. */ describe('leaving Pending', () => { - function transitions(): { status: PaymentLinkPaymentStatus }[] { - return paymentLinkPaymentRepo.update.mock.calls.map( - ([, values]) => values as { status: PaymentLinkPaymentStatus }, - ); - } - it('should expire through a conditional update rather than a status read', async () => { await service.expirePayment(payment({ id: 7 })); - expect(paymentLinkPaymentRepo.update).toHaveBeenCalledWith( - { id: 7, status: PaymentLinkPaymentStatus.PENDING }, - { status: PaymentLinkPaymentStatus.EXPIRED }, - ); - expect(transitions()).toHaveLength(1); + expect(transitions()).toEqual([ + [{ id: 7, status: PaymentLinkPaymentStatus.PENDING }, { status: PaymentLinkPaymentStatus.EXPIRED }], + ]); }); it('should not expire a second time when another process took the transition', async () => { - paymentLinkPaymentRepo.update.mockResolvedValue({ affected: 0 } as never); + row.status = PaymentLinkPaymentStatus.EXPIRED; await service.expirePayment(payment({ id: 7, link: { webhookUrl: 'https://merchant.example/hook' } as never })); @@ -344,15 +382,14 @@ describe('PaymentLinkPaymentService', () => { it('should cancel through the same transition', async () => { await service.cancelByPayment(payment({ id: 7 })); - expect(paymentLinkPaymentRepo.update).toHaveBeenCalledWith( - { id: 7, status: PaymentLinkPaymentStatus.PENDING }, - { status: PaymentLinkPaymentStatus.CANCELLED }, - ); + expect(transitions()).toEqual([ + [{ id: 7, status: PaymentLinkPaymentStatus.PENDING }, { status: PaymentLinkPaymentStatus.CANCELLED }], + ]); expect(paymentQuoteService.cancelAllForPayment).toHaveBeenCalledTimes(1); }); it('should not cancel a payment the worker expired in between', async () => { - paymentLinkPaymentRepo.update.mockResolvedValue({ affected: 0 } as never); + row.status = PaymentLinkPaymentStatus.EXPIRED; await service.cancelByPayment(payment({ id: 7, link: { webhookUrl: 'https://merchant.example/hook' } as never })); @@ -361,6 +398,66 @@ describe('PaymentLinkPaymentService', () => { expect(paymentLinkPaymentRepo.save).not.toHaveBeenCalled(); }); + /** + * What a transition costs when it commits on its own: the row leaves `Pending` and the effects + * that belong to it do not follow. `processExpiredPayments` asks for `Pending`, so such a row + * is out of reach of every job — it is not a delayed repair but a permanent half-state. + */ + describe('a caller that stops between the transition and its effects', () => { + /** The payments the expiry job would find on its next run. */ + function pendingPayments(): PaymentLinkPayment[] { + return row.status === PaymentLinkPaymentStatus.PENDING ? [row] : []; + } + + beforeEach(() => { + // The real job runs here: what makes the half-state permanent is its own query. + new ConfigService(GetConfig()); + paymentLinkPaymentRepo.find.mockImplementation(async () => pendingPayments()); + }); + + it('should ask for nothing but Pending, which is why a half-state is out of its reach', async () => { + await service.processExpiredPayments(); + + expect(paymentLinkPaymentRepo.find).toHaveBeenCalledWith( + expect.objectContaining({ + where: expect.objectContaining({ status: PaymentLinkPaymentStatus.PENDING }), + }), + ); + }); + + it('should leave the payment where the next run of the job picks it up', async () => { + paymentQuoteService.cancelAllForPayment.mockRejectedValue(new Error('connection reset')); + + await expect(service.processExpiredPayments()).rejects.toThrow('connection reset'); + + // Not expired, and its quotes are not cancelled either: both or neither. + expect(row.status).toEqual(PaymentLinkPaymentStatus.PENDING); + expect(paymentActivationService.closeAllForPayment).not.toHaveBeenCalled(); + + // And the next run finds it, which is the property the whole transition rests on. + paymentQuoteService.cancelAllForPayment.mockResolvedValue(undefined); + await service.processExpiredPayments(); + + expect(row.status).toEqual(PaymentLinkPaymentStatus.EXPIRED); + expect(paymentActivationService.closeAllForPayment).toHaveBeenCalledTimes(1); + }); + + it('should have cancelled the quotes before anything a merchant can hold up', async () => { + // The webhook is the one effect that stays outside the transaction, so it must be the last + // one: a payment whose merchant call fails is finished in the database all the same. + paymentLinkPaymentRepo.save.mockRejectedValue(new Error('merchant unreachable')); + + await expect(service.processExpiredPayments()).rejects.toThrow('merchant unreachable'); + + expect(row.status).toEqual(PaymentLinkPaymentStatus.EXPIRED); + expect(paymentQuoteService.cancelAllForPayment).toHaveBeenCalledTimes(1); + expect(paymentActivationService.closeAllForPayment).toHaveBeenCalledTimes(1); + + // Nothing left over: the job does not see it again, and does not have to. + expect(pendingPayments()).toEqual([]); + }); + }); + /** The third way out of `Pending`, reached from a request path and from checkTxConfirmations. */ describe('completing on a quote', () => { function completing(values: Partial = {}): PaymentLinkPayment { @@ -381,15 +478,26 @@ describe('PaymentLinkPaymentService', () => { it('should complete a SINGLE payment through the transition', async () => { await service['handleQuoteChange'](completing(), quote); - expect(paymentLinkPaymentRepo.update).toHaveBeenCalledWith( - { id: 7, status: PaymentLinkPaymentStatus.PENDING }, - { status: PaymentLinkPaymentStatus.COMPLETED }, - ); + expect(transitions()).toEqual([ + [{ id: 7, status: PaymentLinkPaymentStatus.PENDING }, { status: PaymentLinkPaymentStatus.COMPLETED }], + [7, { txCount: 1 }], + ]); expect(paymentLinkPaymentRepo.save).toHaveBeenCalledTimes(1); }); + it('should carry the counted quotes into the transition, not only into the save after it', async () => { + // A completed payment is looked at by no job, so a count left behind by a caller that + // stopped after the transition would stay wrong for good. + paymentLinkPaymentRepo.save.mockRejectedValue(new Error('merchant unreachable')); + + await expect(service['handleQuoteChange'](completing(), quote)).rejects.toThrow('merchant unreachable'); + + expect(row.status).toEqual(PaymentLinkPaymentStatus.COMPLETED); + expect(row.txCount).toEqual(1); + }); + it('should not complete it again when another process got there first', async () => { - paymentLinkPaymentRepo.update.mockResolvedValue({ affected: 0 } as never); + row.status = PaymentLinkPaymentStatus.COMPLETED; await service['handleQuoteChange'](completing(), quote); @@ -401,7 +509,7 @@ describe('PaymentLinkPaymentService', () => { // but one from recording the quote it counted. await service['handleQuoteChange'](completing({ mode: PaymentLinkPaymentMode.MULTIPLE }), quote); - expect(paymentLinkPaymentRepo.update).not.toHaveBeenCalled(); + expect(transitions()).toEqual([]); expect(paymentLinkPaymentRepo.save).toHaveBeenCalledTimes(1); }); }); diff --git a/src/subdomains/core/payment-link/services/payment-activation.service.ts b/src/subdomains/core/payment-link/services/payment-activation.service.ts index dd5bd9b967..06ccb12137 100644 --- a/src/subdomains/core/payment-link/services/payment-activation.service.ts +++ b/src/subdomains/core/payment-link/services/payment-activation.service.ts @@ -1,311 +1,314 @@ -import { BadRequestException, ConflictException, Injectable, NotFoundException } from '@nestjs/common'; -import { Config } from 'src/config/config'; -import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.enum'; -import { CryptoService } from 'src/integration/blockchain/shared/services/crypto.service'; -import { LnBitsWalletPaymentParamsDto } from 'src/integration/lightning/dto/lnbits.dto'; -import { LightningClient } from 'src/integration/lightning/lightning-client'; -import { LightningHelper } from 'src/integration/lightning/lightning-helper'; -import { LightningService } from 'src/integration/lightning/services/lightning.service'; -import { Asset } from 'src/shared/models/asset/asset.entity'; -import { AssetService } from 'src/shared/models/asset/asset.service'; -import { DfxLogger } from 'src/shared/services/dfx-logger'; -import { Util } from 'src/shared/utils/util'; -import { C2BPaymentLinkService } from 'src/subdomains/core/payment-link/services/c2b-payment-link.service'; -import { Equal, LessThan, Not } from 'typeorm'; -import { TransferInfo } from '../dto/payment-link.dto'; -import { PaymentActivation } from '../entities/payment-activation.entity'; -import { PaymentLinkPayment } from '../entities/payment-link-payment.entity'; -import { PaymentQuote } from '../entities/payment-quote.entity'; -import { PaymentActivationStatus, PaymentLinkPaymentMode, PaymentStandard } from '../enums'; -import { PaymentActivationRepository } from '../repositories/payment-activation.repository'; -import { PaymentBalanceService } from './payment-balance.service'; -import { PaymentQuoteService } from './payment-quote.service'; - -@Injectable() -export class PaymentActivationService { - private readonly logger = new DfxLogger(PaymentActivationService); - - private readonly client: LightningClient; - - constructor( - readonly lightningService: LightningService, - private readonly paymentActivationRepo: PaymentActivationRepository, - private readonly paymentQuoteService: PaymentQuoteService, - private readonly paymentBalanceService: PaymentBalanceService, - private readonly assetService: AssetService, - private readonly cryptoService: CryptoService, - private readonly c2bPaymentLinkService: C2BPaymentLinkService, - ) { - this.client = lightningService.getDefaultClient(); - } - - async close(activation: PaymentActivation): Promise { - await this.paymentActivationRepo.update( - { id: activation.id, status: Not(PaymentActivationStatus.CLOSED) }, - { status: PaymentActivationStatus.CLOSED }, - ); - } - - async closeAllForPayment(paymentId: number): Promise { - await this.paymentActivationRepo.update( - { payment: { id: paymentId }, status: Not(PaymentActivationStatus.CLOSED) }, - { status: PaymentActivationStatus.CLOSED }, - ); - } - - async closeAllForQuote(quoteId: number): Promise { - await this.paymentActivationRepo.update( - { quote: { id: quoteId }, status: Not(PaymentActivationStatus.CLOSED) }, - { status: PaymentActivationStatus.CLOSED }, - ); - } - - async getActivationByTxId(txHash: string): Promise { - return this.paymentActivationRepo.findOne({ - where: { paymentHash: Equal(txHash), status: PaymentActivationStatus.OPEN }, - relations: { quote: { payment: true } }, - }); - } - - async deleteActivation(activation: PaymentActivation): Promise { - await this.paymentActivationRepo.delete(activation.id); - } - - // --- HANDLE PENDING ACTIVATIONS --- // - async processExpiredActivations(): Promise { - const maxDate = Util.secondsBefore(Config.payment.timeoutDelay); - - await this.paymentActivationRepo.update( - { - status: PaymentActivationStatus.OPEN, - expiryDate: LessThan(maxDate), - }, - { status: PaymentActivationStatus.CLOSED }, - ); - } - - // --- CREATE ACTIVATIONS --- // - - async doCreateRequest(pendingPayment: PaymentLinkPayment, transferInfo: TransferInfo): Promise { - const actualQuote = await this.paymentQuoteService.getActualQuote(pendingPayment, transferInfo); - if (!actualQuote) throw new NotFoundException(`No matching actual quote found`); - - if (transferInfo.quoteUniqueId) { - const transferAmount = actualQuote.getTransferAmountFor(transferInfo.method, transferInfo.asset)?.amount; - - if (!transferAmount) throw new BadRequestException(`Invalid method or asset`); - - transferInfo.amount = transferAmount; - } - - const expiryDate = new Date(Math.min(pendingPayment.expiryDate.getTime(), actualQuote.expiryDate.getTime())); - - const expirySec = Util.secondsDiff(new Date(), expiryDate); - if (expirySec < 1) throw new BadRequestException('Quote is expired'); - - const activations = await this.getExistingActivations(transferInfo); - if ( - actualQuote.standard === PaymentStandard.PAY_TO_ADDRESS && - activations.some( - (a) => - a.standard === PaymentStandard.PAY_TO_ADDRESS && - (a.quote.id !== actualQuote.id || pendingPayment.mode === PaymentLinkPaymentMode.MULTIPLE), - ) - ) - throw new ConflictException('Duplicate payment request'); - - return ( - activations.find((a) => a.quote.id === actualQuote.id) ?? - this.createNewPaymentActivationRequest( - pendingPayment, - actualQuote, - transferInfo, - expirySec, - expiryDate, - actualQuote.standard, - ) - ); - } - - private async getExistingActivations(transferInfo: TransferInfo): Promise { - return this.paymentActivationRepo.find({ - where: { - status: PaymentActivationStatus.OPEN, - amount: transferInfo.amount, - method: transferInfo.method, - asset: { uniqueName: `${transferInfo.method}/${transferInfo.asset}` }, - }, - relations: { - payment: true, - quote: true, - }, - }); - } - - private async createNewPaymentActivationRequest( - payment: PaymentLinkPayment, - quote: PaymentQuote, - transferInfo: TransferInfo, - expirySec: number, - expiryDate: Date, - standard: PaymentStandard, - ): Promise { - const { paymentRequest, paymentHash } = await this.createBlockchainRequest(payment, transferInfo, expirySec, quote); - - return this.savePaymentActivationRequest( - payment, - quote, - paymentRequest, - paymentHash, - transferInfo, - expiryDate, - standard, - ); - } - - private async createBlockchainRequest( - payment: PaymentLinkPayment, - transferInfo: TransferInfo, - expirySec: number, - quote: PaymentQuote, - ): Promise<{ paymentRequest: string; paymentHash?: string }> { - switch (transferInfo.method) { - case Blockchain.LIGHTNING: - return this.createLightningRequest(payment, transferInfo, expirySec); - - case Blockchain.BITCOIN: - case Blockchain.FIRO: - case Blockchain.MONERO: - case Blockchain.ZANO: - case Blockchain.ETHEREUM: - case Blockchain.SEPOLIA: - case Blockchain.ARBITRUM: - case Blockchain.OPTIMISM: - case Blockchain.BASE: - case Blockchain.GNOSIS: - case Blockchain.POLYGON: - case Blockchain.BINANCE_SMART_CHAIN: - case Blockchain.SOLANA: - case Blockchain.TRON: - case Blockchain.CARDANO: - case Blockchain.INTERNET_COMPUTER: { - const address = this.paymentBalanceService.getDepositAddress(transferInfo.method); - if (address) return this.createPaymentRequest(address, transferInfo, 'DFX Payment'); - - break; - } - - case Blockchain.KUCOIN_PAY: - case Blockchain.BINANCE_PAY: - return this.createC2BPaymentRequest(payment, transferInfo, quote); - } - - throw new BadRequestException(`Invalid method ${transferInfo.method}`); - } - - private async createLightningRequest( - payment: PaymentLinkPayment, - transferInfo: TransferInfo, - expirySec: number, - ): Promise<{ paymentRequest: string; paymentHash: string }> { - const lnurlpAddress = await this.getDepositLnurlpAddress(payment); - if (!lnurlpAddress) throw new BadRequestException('Deposit LNURLp Address not found'); - - const uniqueId = payment.uniqueId; - const uniqueIdSignature = Util.createSign(uniqueId, Config.blockchain.lightning.lnbits.signingPrivKey); - - const walletPaymentParams: LnBitsWalletPaymentParamsDto = { - amount: LightningHelper.btcToSat(transferInfo.amount), - memo: payment.memo, - expirySec: expirySec, - webhook: `${Config.url()}/payIn/lnurlpPayment/${uniqueId}`, - extra: { - link: lnurlpAddress, - signature: uniqueIdSignature, - }, - }; - - const paymentRequest = await this.client.getLnBitsWalletPayment(walletPaymentParams).then((r) => r.pr); - const paymentHash = LightningHelper.getPaymentHashOfInvoice(paymentRequest); - - return { paymentRequest, paymentHash }; - } - - private async getDepositLnurlpAddress(pendingPayment: PaymentLinkPayment): Promise { - try { - const depositAddress = pendingPayment.link.route.deposit.address; - - if (!depositAddress.startsWith('LNURL')) { - this.logger.error( - `Lightning transaction: Deposit address ${depositAddress} is not a LNURL address for payment link ${pendingPayment.link.uniqueId}`, - ); - return; - } - - const decodedDepositAddress = LightningHelper.decodeLnurl(depositAddress); - const paths = decodedDepositAddress.split('/'); - return paths[paths.length - 1]; - } catch (e) { - this.logger.error( - `Lightning transaction: Cannot get LNURLp address for payment link ${pendingPayment.link.id}`, - e, - ); - } - } - - private async createPaymentRequest( - address: string, - transferInfo: TransferInfo, - label?: string, - ): Promise<{ paymentRequest: string; paymentHash?: string }> { - const asset = await this.getAssetByInfo(transferInfo); - - const paymentRequest = await this.cryptoService.getPaymentRequest(true, asset, address, transferInfo.amount, label); - return { paymentRequest }; - } - - private async createC2BPaymentRequest( - payment: PaymentLinkPayment, - transferInfo: TransferInfo, - quote: PaymentQuote, - ): Promise<{ paymentRequest: string; paymentHash: string }> { - const order = await this.c2bPaymentLinkService.createOrder(payment, transferInfo, quote); - return { paymentRequest: order.paymentRequest, paymentHash: order.providerOrderId }; - } - - private async savePaymentActivationRequest( - payment: PaymentLinkPayment, - quote: PaymentQuote, - paymentRequest: string, - paymentHash: string, - transferInfo: TransferInfo, - expiryDate: Date, - standard: PaymentStandard, - ): Promise { - const asset = await this.getAssetByInfo(transferInfo); - - const newPaymentActivation = this.paymentActivationRepo.create({ - status: PaymentActivationStatus.OPEN, - method: transferInfo.method, - amount: transferInfo.amount, - asset, - paymentRequest, - paymentHash, - expiryDate, - standard, - payment, - quote, - }); - - return this.paymentActivationRepo.save(newPaymentActivation); - } - - private async getAssetByInfo(transferInfo: TransferInfo): Promise { - const uniqueName = `${transferInfo.method}/${transferInfo.asset}`; - - const asset = await this.assetService.getAssetByUniqueName(uniqueName); - if (!asset) throw new NotFoundException(`Asset ${uniqueName} not found`); - - return asset; - } -} +import { BadRequestException, ConflictException, Injectable, NotFoundException } from '@nestjs/common'; +import { Config } from 'src/config/config'; +import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.enum'; +import { CryptoService } from 'src/integration/blockchain/shared/services/crypto.service'; +import { LnBitsWalletPaymentParamsDto } from 'src/integration/lightning/dto/lnbits.dto'; +import { LightningClient } from 'src/integration/lightning/lightning-client'; +import { LightningHelper } from 'src/integration/lightning/lightning-helper'; +import { LightningService } from 'src/integration/lightning/services/lightning.service'; +import { Asset } from 'src/shared/models/asset/asset.entity'; +import { AssetService } from 'src/shared/models/asset/asset.service'; +import { DfxLogger } from 'src/shared/services/dfx-logger'; +import { Util } from 'src/shared/utils/util'; +import { C2BPaymentLinkService } from 'src/subdomains/core/payment-link/services/c2b-payment-link.service'; +import { EntityManager, Equal, LessThan, Not, Repository } from 'typeorm'; +import { TransferInfo } from '../dto/payment-link.dto'; +import { PaymentActivation } from '../entities/payment-activation.entity'; +import { PaymentLinkPayment } from '../entities/payment-link-payment.entity'; +import { PaymentQuote } from '../entities/payment-quote.entity'; +import { PaymentActivationStatus, PaymentLinkPaymentMode, PaymentStandard } from '../enums'; +import { PaymentActivationRepository } from '../repositories/payment-activation.repository'; +import { PaymentBalanceService } from './payment-balance.service'; +import { PaymentQuoteService } from './payment-quote.service'; + +@Injectable() +export class PaymentActivationService { + private readonly logger = new DfxLogger(PaymentActivationService); + + private readonly client: LightningClient; + + constructor( + readonly lightningService: LightningService, + private readonly paymentActivationRepo: PaymentActivationRepository, + private readonly paymentQuoteService: PaymentQuoteService, + private readonly paymentBalanceService: PaymentBalanceService, + private readonly assetService: AssetService, + private readonly cryptoService: CryptoService, + private readonly c2bPaymentLinkService: C2BPaymentLinkService, + ) { + this.client = lightningService.getDefaultClient(); + } + + async close(activation: PaymentActivation): Promise { + await this.paymentActivationRepo.update( + { id: activation.id, status: Not(PaymentActivationStatus.CLOSED) }, + { status: PaymentActivationStatus.CLOSED }, + ); + } + + /** `manager` runs the closes in the caller's transaction, see `cancelAllForPayment` for why. */ + async closeAllForPayment(paymentId: number, manager?: EntityManager): Promise { + const repo: Repository = manager?.getRepository(PaymentActivation) ?? this.paymentActivationRepo; + + await repo.update( + { payment: { id: paymentId }, status: Not(PaymentActivationStatus.CLOSED) }, + { status: PaymentActivationStatus.CLOSED }, + ); + } + + async closeAllForQuote(quoteId: number): Promise { + await this.paymentActivationRepo.update( + { quote: { id: quoteId }, status: Not(PaymentActivationStatus.CLOSED) }, + { status: PaymentActivationStatus.CLOSED }, + ); + } + + async getActivationByTxId(txHash: string): Promise { + return this.paymentActivationRepo.findOne({ + where: { paymentHash: Equal(txHash), status: PaymentActivationStatus.OPEN }, + relations: { quote: { payment: true } }, + }); + } + + async deleteActivation(activation: PaymentActivation): Promise { + await this.paymentActivationRepo.delete(activation.id); + } + + // --- HANDLE PENDING ACTIVATIONS --- // + async processExpiredActivations(): Promise { + const maxDate = Util.secondsBefore(Config.payment.timeoutDelay); + + await this.paymentActivationRepo.update( + { + status: PaymentActivationStatus.OPEN, + expiryDate: LessThan(maxDate), + }, + { status: PaymentActivationStatus.CLOSED }, + ); + } + + // --- CREATE ACTIVATIONS --- // + + async doCreateRequest(pendingPayment: PaymentLinkPayment, transferInfo: TransferInfo): Promise { + const actualQuote = await this.paymentQuoteService.getActualQuote(pendingPayment, transferInfo); + if (!actualQuote) throw new NotFoundException(`No matching actual quote found`); + + if (transferInfo.quoteUniqueId) { + const transferAmount = actualQuote.getTransferAmountFor(transferInfo.method, transferInfo.asset)?.amount; + + if (!transferAmount) throw new BadRequestException(`Invalid method or asset`); + + transferInfo.amount = transferAmount; + } + + const expiryDate = new Date(Math.min(pendingPayment.expiryDate.getTime(), actualQuote.expiryDate.getTime())); + + const expirySec = Util.secondsDiff(new Date(), expiryDate); + if (expirySec < 1) throw new BadRequestException('Quote is expired'); + + const activations = await this.getExistingActivations(transferInfo); + if ( + actualQuote.standard === PaymentStandard.PAY_TO_ADDRESS && + activations.some( + (a) => + a.standard === PaymentStandard.PAY_TO_ADDRESS && + (a.quote.id !== actualQuote.id || pendingPayment.mode === PaymentLinkPaymentMode.MULTIPLE), + ) + ) + throw new ConflictException('Duplicate payment request'); + + return ( + activations.find((a) => a.quote.id === actualQuote.id) ?? + this.createNewPaymentActivationRequest( + pendingPayment, + actualQuote, + transferInfo, + expirySec, + expiryDate, + actualQuote.standard, + ) + ); + } + + private async getExistingActivations(transferInfo: TransferInfo): Promise { + return this.paymentActivationRepo.find({ + where: { + status: PaymentActivationStatus.OPEN, + amount: transferInfo.amount, + method: transferInfo.method, + asset: { uniqueName: `${transferInfo.method}/${transferInfo.asset}` }, + }, + relations: { + payment: true, + quote: true, + }, + }); + } + + private async createNewPaymentActivationRequest( + payment: PaymentLinkPayment, + quote: PaymentQuote, + transferInfo: TransferInfo, + expirySec: number, + expiryDate: Date, + standard: PaymentStandard, + ): Promise { + const { paymentRequest, paymentHash } = await this.createBlockchainRequest(payment, transferInfo, expirySec, quote); + + return this.savePaymentActivationRequest( + payment, + quote, + paymentRequest, + paymentHash, + transferInfo, + expiryDate, + standard, + ); + } + + private async createBlockchainRequest( + payment: PaymentLinkPayment, + transferInfo: TransferInfo, + expirySec: number, + quote: PaymentQuote, + ): Promise<{ paymentRequest: string; paymentHash?: string }> { + switch (transferInfo.method) { + case Blockchain.LIGHTNING: + return this.createLightningRequest(payment, transferInfo, expirySec); + + case Blockchain.BITCOIN: + case Blockchain.FIRO: + case Blockchain.MONERO: + case Blockchain.ZANO: + case Blockchain.ETHEREUM: + case Blockchain.SEPOLIA: + case Blockchain.ARBITRUM: + case Blockchain.OPTIMISM: + case Blockchain.BASE: + case Blockchain.GNOSIS: + case Blockchain.POLYGON: + case Blockchain.BINANCE_SMART_CHAIN: + case Blockchain.SOLANA: + case Blockchain.TRON: + case Blockchain.CARDANO: + case Blockchain.INTERNET_COMPUTER: { + const address = this.paymentBalanceService.getDepositAddress(transferInfo.method); + if (address) return this.createPaymentRequest(address, transferInfo, 'DFX Payment'); + + break; + } + + case Blockchain.KUCOIN_PAY: + case Blockchain.BINANCE_PAY: + return this.createC2BPaymentRequest(payment, transferInfo, quote); + } + + throw new BadRequestException(`Invalid method ${transferInfo.method}`); + } + + private async createLightningRequest( + payment: PaymentLinkPayment, + transferInfo: TransferInfo, + expirySec: number, + ): Promise<{ paymentRequest: string; paymentHash: string }> { + const lnurlpAddress = await this.getDepositLnurlpAddress(payment); + if (!lnurlpAddress) throw new BadRequestException('Deposit LNURLp Address not found'); + + const uniqueId = payment.uniqueId; + const uniqueIdSignature = Util.createSign(uniqueId, Config.blockchain.lightning.lnbits.signingPrivKey); + + const walletPaymentParams: LnBitsWalletPaymentParamsDto = { + amount: LightningHelper.btcToSat(transferInfo.amount), + memo: payment.memo, + expirySec: expirySec, + webhook: `${Config.url()}/payIn/lnurlpPayment/${uniqueId}`, + extra: { + link: lnurlpAddress, + signature: uniqueIdSignature, + }, + }; + + const paymentRequest = await this.client.getLnBitsWalletPayment(walletPaymentParams).then((r) => r.pr); + const paymentHash = LightningHelper.getPaymentHashOfInvoice(paymentRequest); + + return { paymentRequest, paymentHash }; + } + + private async getDepositLnurlpAddress(pendingPayment: PaymentLinkPayment): Promise { + try { + const depositAddress = pendingPayment.link.route.deposit.address; + + if (!depositAddress.startsWith('LNURL')) { + this.logger.error( + `Lightning transaction: Deposit address ${depositAddress} is not a LNURL address for payment link ${pendingPayment.link.uniqueId}`, + ); + return; + } + + const decodedDepositAddress = LightningHelper.decodeLnurl(depositAddress); + const paths = decodedDepositAddress.split('/'); + return paths[paths.length - 1]; + } catch (e) { + this.logger.error( + `Lightning transaction: Cannot get LNURLp address for payment link ${pendingPayment.link.id}`, + e, + ); + } + } + + private async createPaymentRequest( + address: string, + transferInfo: TransferInfo, + label?: string, + ): Promise<{ paymentRequest: string; paymentHash?: string }> { + const asset = await this.getAssetByInfo(transferInfo); + + const paymentRequest = await this.cryptoService.getPaymentRequest(true, asset, address, transferInfo.amount, label); + return { paymentRequest }; + } + + private async createC2BPaymentRequest( + payment: PaymentLinkPayment, + transferInfo: TransferInfo, + quote: PaymentQuote, + ): Promise<{ paymentRequest: string; paymentHash: string }> { + const order = await this.c2bPaymentLinkService.createOrder(payment, transferInfo, quote); + return { paymentRequest: order.paymentRequest, paymentHash: order.providerOrderId }; + } + + private async savePaymentActivationRequest( + payment: PaymentLinkPayment, + quote: PaymentQuote, + paymentRequest: string, + paymentHash: string, + transferInfo: TransferInfo, + expiryDate: Date, + standard: PaymentStandard, + ): Promise { + const asset = await this.getAssetByInfo(transferInfo); + + const newPaymentActivation = this.paymentActivationRepo.create({ + status: PaymentActivationStatus.OPEN, + method: transferInfo.method, + amount: transferInfo.amount, + asset, + paymentRequest, + paymentHash, + expiryDate, + standard, + payment, + quote, + }); + + return this.paymentActivationRepo.save(newPaymentActivation); + } + + private async getAssetByInfo(transferInfo: TransferInfo): Promise { + const uniqueName = `${transferInfo.method}/${transferInfo.asset}`; + + const asset = await this.assetService.getAssetByUniqueName(uniqueName); + if (!asset) throw new NotFoundException(`Asset ${uniqueName} not found`); + + return asset; + } +} diff --git a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts index a2abd94281..0df2583175 100644 --- a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts @@ -10,7 +10,7 @@ import { AsyncMap } from 'src/shared/utils/async-map'; import { Util } from 'src/shared/utils/util'; import { C2BWebhookResult } from 'src/subdomains/core/payment-link/share/c2b-payment-link.provider'; import { CryptoInput } from 'src/subdomains/supporting/payin/entities/crypto-input.entity'; -import { In, IsNull, LessThan, MoreThan, Not } from 'typeorm'; +import { EntityManager, In, IsNull, LessThan, MoreThan, Not } from 'typeorm'; import { isSellRoute } from '../../sell-crypto/route/sell.entity'; import { CreatePaymentLinkPaymentDto } from '../dto/create-payment-link-payment.dto'; import { PaymentLinkEvmPaymentDto, PaymentLinkHexResultDto, TransferInfo } from '../dto/payment-link.dto'; @@ -115,16 +115,18 @@ export class PaymentLinkPaymentService { } async expirePayment(payment: PaymentLinkPayment): Promise { - if (!(await this.takePendingTransition(payment, PaymentLinkPaymentStatus.EXPIRED))) return; + const taken = await this.takePendingTransition(payment, PaymentLinkPaymentStatus.EXPIRED, (manager) => + this.cancelQuotesForPayment(payment.id, manager), + ); + if (!taken) return; await this.doSave(payment.expire(), true); - await this.cancelQuotesForPayment(payment); } /** - * Moves a payment out of `Pending` in one statement, and answers whether THIS caller is the one - * that moved it. Everything that follows a transition — the merchant webhook, the quote - * cancellations, the activation closes — belongs to the caller that gets `true`. + * Moves a payment out of `Pending` together with the database effects that belong to that + * transition, and answers whether THIS caller is the one that moved it. Everything that follows + * — the merchant webhook, the deliveries in `doSave` — belongs to the caller that gets `true`. * * The read-then-write this replaces was safe while everything ran in one process. It is not any * more: `processExpiredPayments` is a `Worker` job, while the expiry timers `createPayment` arms @@ -137,14 +139,36 @@ export class PaymentLinkPaymentService { * `Pending` and reports it as the affected row, and every other caller gets nothing. That holds * for any number of processes and for every path into the transition, which is why it sits here * rather than at the call sites. + * + * `effects` runs in the same transaction as that statement, so the row leaves `Pending` only if + * they leave with it. A statement committing on its own would be worse than the double run it + * prevents: a caller dying between the two would leave a payment out of `Pending` with its + * quotes still open, and nothing looks for that — `processExpiredPayments` asks for `Pending` + * and would never see the row again. Rolled back, the payment stays exactly where the next run + * of the job, or of the timer, picks it up. + * + * What stays outside is what a transaction must not hold open: the merchant webhook and the + * process-local deliveries in `doSave`. Those cost a notification when they are lost, not a row + * that no one reconciles — and the webhook is best-effort by construction (see + * `PaymentWebhookService.sendWebhook`). */ - private async takePendingTransition(payment: PaymentLinkPayment, status: PaymentLinkPaymentStatus): Promise { - const { affected } = await this.paymentLinkPaymentRepo.update( - { id: payment.id, status: PaymentLinkPaymentStatus.PENDING }, - { status }, - ); - - return affected > 0; + private async takePendingTransition( + payment: PaymentLinkPayment, + status: PaymentLinkPaymentStatus, + effects: (manager: EntityManager) => Promise, + ): Promise { + return this.paymentLinkPaymentRepo.manager.transaction(async (manager) => { + const { affected } = await manager.update( + PaymentLinkPayment, + { id: payment.id, status: PaymentLinkPaymentStatus.PENDING }, + { status }, + ); + if (!affected) return false; + + await effects(manager); + + return true; + }); } async checkTxConfirmations(): Promise { @@ -505,10 +529,12 @@ export class PaymentLinkPaymentService { async cancelByPayment(payment: PaymentLinkPayment): Promise { // Both callers reach here from a payment they read as `Pending`, and the worker can expire // that same payment in between. Whoever the transition lets through sends the webhook. - if (!(await this.takePendingTransition(payment, PaymentLinkPaymentStatus.CANCELLED))) return; + const taken = await this.takePendingTransition(payment, PaymentLinkPaymentStatus.CANCELLED, (manager) => + this.cancelQuotesForPayment(payment.id, manager), + ); + if (!taken) return; await this.doSave(payment.cancel(), true); - await this.cancelQuotesForPayment(payment); } async deletePayment(payment: PaymentLinkPayment): Promise { @@ -526,9 +552,10 @@ export class PaymentLinkPaymentService { await this.paymentLinkPaymentRepo.delete(payment.id); } - private async cancelQuotesForPayment(payment: PaymentLinkPayment): Promise { - await this.paymentQuoteService.cancelAllForPayment(payment.id); - await this.paymentActivationService.closeAllForPayment(payment.id); + /** The database effects of leaving `Pending`, run on the manager of the transition's transaction. */ + private async cancelQuotesForPayment(paymentId: number, manager: EntityManager): Promise { + await this.paymentQuoteService.cancelAllForPayment(paymentId, manager); + await this.paymentActivationService.closeAllForPayment(paymentId, manager); } // --- HANDLE CALLBACKS --- // @@ -639,13 +666,24 @@ export class PaymentLinkPaymentService { const isPaymentComplete = PaymentQuoteTxStates.indexOf(quote.status) >= PaymentQuoteTxStates.indexOf(minCompletionStatus); if (isPaymentComplete) { - payment.txCount = await this.paymentQuoteService.getCompletedQuoteCount(payment, minCompletionStatus); + const txCount = await this.paymentQuoteService.getCompletedQuoteCount(payment, minCompletionStatus); + payment.txCount = txCount; // The status read above is the same read-then-write as in expirePayment, and this one is // reached from request paths as well as from checkTxConfirmations. A `MULTIPLE` payment // stays `Pending` and has no transition to take: it only counts a quote. if (payment.mode === PaymentLinkPaymentMode.SINGLE) { - if (!(await this.takePendingTransition(payment, PaymentLinkPaymentStatus.COMPLETED))) return; + const taken = await this.takePendingTransition( + payment, + PaymentLinkPaymentStatus.COMPLETED, + // The count belongs to the transition: `doSave` below is the only other thing that + // writes it, and a completed payment no job looks at again would keep whatever count it + // had when the caller stopped. The activations are already closed above. + async (manager) => { + await manager.update(PaymentLinkPayment, payment.id, { txCount }); + }, + ); + if (!taken) return; payment.complete(); } diff --git a/src/subdomains/core/payment-link/services/payment-quote.service.ts b/src/subdomains/core/payment-link/services/payment-quote.service.ts index aa1577e99d..6fc48d9792 100644 --- a/src/subdomains/core/payment-link/services/payment-quote.service.ts +++ b/src/subdomains/core/payment-link/services/payment-quote.service.ts @@ -20,7 +20,7 @@ import { C2BPaymentLinkService } from 'src/subdomains/core/payment-link/services import { PaymentBalanceService } from 'src/subdomains/core/payment-link/services/payment-balance.service'; import { PaymentLinkFeeService } from 'src/subdomains/core/payment-link/services/payment-link-fee.service'; import { PriceValidity, PricingService } from 'src/subdomains/supporting/pricing/services/pricing.service'; -import { Equal, In, LessThan } from 'typeorm'; +import { EntityManager, Equal, In, LessThan, Repository } from 'typeorm'; import { TransferAmount, TransferAmountAsset, TransferInfo } from '../dto/payment-link.dto'; import { PaymentLinkPayment } from '../entities/payment-link-payment.entity'; import { PaymentLink } from '../entities/payment-link.entity'; @@ -173,13 +173,20 @@ export class PaymentQuoteService { }); } - async cancelAllForPayment(paymentId: number): Promise { - const actualQuotes = await this.paymentQuoteRepo.find({ + /** + * `manager` runs the cancellations in the caller's transaction. The caller is the payment leaving + * `Pending` (see `PaymentLinkPaymentService.takePendingTransition`), and these quotes have to + * leave `Actual` with it or not at all: a payment out of `Pending` is not looked at again. + */ + async cancelAllForPayment(paymentId: number, manager?: EntityManager): Promise { + const repo: Repository = manager?.getRepository(PaymentQuote) ?? this.paymentQuoteRepo; + + const actualQuotes = await repo.find({ where: { payment: { id: paymentId }, status: PaymentQuoteStatus.ACTUAL }, }); for (const actualQuote of actualQuotes) { - await this.paymentQuoteRepo.save(actualQuote.cancel()); + await repo.save(actualQuote.cancel()); } } From 342d19f8d6d87b71ed2f08a40a16671f7f053698 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:34 +0200 Subject: [PATCH 61/86] Pin sdk-metrics to the version the exporters carry npm resolved the caret range to a version above the one the six exporter packages depend on, so two copies of @opentelemetry/sdk-metrics sat in the tree: the meter provider was built against one and read through the other. The lockfile now holds a single copy. --- package-lock.json | 122 +++++----------------------------------------- package.json | 2 +- src/tracing.ts | 5 ++ 3 files changed, 19 insertions(+), 110 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6fc98b7c19..c46015f352 100644 --- a/package-lock.json +++ b/package-lock.json @@ -46,7 +46,7 @@ "@opentelemetry/auto-instrumentations-node": "^0.76.0", "@opentelemetry/exporter-metrics-otlp-http": "^0.218.0", "@opentelemetry/exporter-trace-otlp-http": "^0.218.0", - "@opentelemetry/sdk-metrics": "^2.7.1", + "@opentelemetry/sdk-metrics": "2.7.1", "@opentelemetry/sdk-node": "^0.218.0", "@opentelemetry/sdk-trace-base": "^2.7.1", "@railgun-community/engine": "^9.4.0", @@ -7812,22 +7812,6 @@ "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, - "node_modules/@opentelemetry/exporter-metrics-otlp-grpc/node_modules/@opentelemetry/sdk-metrics": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.7.1.tgz", - "integrity": "sha512-MpDJdkiFDs3Pm1RHO3KByuZbuBdJEXEAkiC0+yJdsZGVCdf1RpHR6n+LHDcS7ffmfrt5kVCzJSCfm4z2C7v0uQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.7.1", - "@opentelemetry/resources": "2.7.1" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.9.0 <1.10.0" - } - }, "node_modules/@opentelemetry/exporter-metrics-otlp-grpc/node_modules/cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", @@ -7916,22 +7900,6 @@ "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, - "node_modules/@opentelemetry/exporter-metrics-otlp-http/node_modules/@opentelemetry/sdk-metrics": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.7.1.tgz", - "integrity": "sha512-MpDJdkiFDs3Pm1RHO3KByuZbuBdJEXEAkiC0+yJdsZGVCdf1RpHR6n+LHDcS7ffmfrt5kVCzJSCfm4z2C7v0uQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.7.1", - "@opentelemetry/resources": "2.7.1" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.9.0 <1.10.0" - } - }, "node_modules/@opentelemetry/exporter-metrics-otlp-proto": { "version": "0.218.0", "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-metrics-otlp-proto/-/exporter-metrics-otlp-proto-0.218.0.tgz", @@ -7983,22 +7951,6 @@ "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, - "node_modules/@opentelemetry/exporter-metrics-otlp-proto/node_modules/@opentelemetry/sdk-metrics": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.7.1.tgz", - "integrity": "sha512-MpDJdkiFDs3Pm1RHO3KByuZbuBdJEXEAkiC0+yJdsZGVCdf1RpHR6n+LHDcS7ffmfrt5kVCzJSCfm4z2C7v0uQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.7.1", - "@opentelemetry/resources": "2.7.1" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.9.0 <1.10.0" - } - }, "node_modules/@opentelemetry/exporter-prometheus": { "version": "0.218.0", "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-prometheus/-/exporter-prometheus-0.218.0.tgz", @@ -8048,22 +8000,6 @@ "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, - "node_modules/@opentelemetry/exporter-prometheus/node_modules/@opentelemetry/sdk-metrics": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.7.1.tgz", - "integrity": "sha512-MpDJdkiFDs3Pm1RHO3KByuZbuBdJEXEAkiC0+yJdsZGVCdf1RpHR6n+LHDcS7ffmfrt5kVCzJSCfm4z2C7v0uQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.7.1", - "@opentelemetry/resources": "2.7.1" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.9.0 <1.10.0" - } - }, "node_modules/@opentelemetry/exporter-trace-otlp-grpc": { "version": "0.218.0", "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-grpc/-/exporter-trace-otlp-grpc-0.218.0.tgz", @@ -11760,22 +11696,6 @@ "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, - "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/sdk-metrics": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.7.1.tgz", - "integrity": "sha512-MpDJdkiFDs3Pm1RHO3KByuZbuBdJEXEAkiC0+yJdsZGVCdf1RpHR6n+LHDcS7ffmfrt5kVCzJSCfm4z2C7v0uQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.7.1", - "@opentelemetry/resources": "2.7.1" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.9.0 <1.10.0" - } - }, "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/sdk-trace-base": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.7.1.tgz", @@ -12023,13 +11943,13 @@ } }, "node_modules/@opentelemetry/sdk-metrics": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.10.0.tgz", - "integrity": "sha512-t6r1VSvXNtSDnPXU1FbZeetJb7yyovHmgu0wRSoftxtE0g2rSNhQZQUy69sRUCL+iioJpX8SN/S6wq6ZtvLySQ==", + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.7.1.tgz", + "integrity": "sha512-MpDJdkiFDs3Pm1RHO3KByuZbuBdJEXEAkiC0+yJdsZGVCdf1RpHR6n+LHDcS7ffmfrt5kVCzJSCfm4z2C7v0uQ==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "2.10.0", - "@opentelemetry/resources": "2.10.0" + "@opentelemetry/core": "2.7.1", + "@opentelemetry/resources": "2.7.1" }, "engines": { "node": "^18.19.0 || >=20.6.0" @@ -12039,9 +11959,9 @@ } }, "node_modules/@opentelemetry/sdk-metrics/node_modules/@opentelemetry/core": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.10.0.tgz", - "integrity": "sha512-/wNZ8twnEQQA4HoHu22+vcsdru6pWPWxW+7w+FlxT6Id7PE/WIbZmVKkte+PF72e0F2dnImFeHD2syyE1Mw6MQ==", + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.7.1.tgz", + "integrity": "sha512-QAqIj32AtK6+pEVNG7EOVxHdE06RP+FM5qpiEJ4RtDcFIqKUZHYhl7/7UY5efhwmwNAg7j8QbJVBLxMerc0+gw==", "license": "Apache-2.0", "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" @@ -12054,12 +11974,12 @@ } }, "node_modules/@opentelemetry/sdk-metrics/node_modules/@opentelemetry/resources": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.10.0.tgz", - "integrity": "sha512-q6MMm2zhggzsHVNbabYwut+a6nbuQQe3URUoxaojM/8K1IBfwwPzvxIjNi2/lI1TFe+fMHMW9MWhrtDLEXEnkA==", + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.7.1.tgz", + "integrity": "sha512-DeT6KKolmC4e/dRQvMQ/RwlnzhaqeiFOXY5ngoOPJ07GgVVKxZOg9EcrNZb5aTzUn+iCrJldAgOfQm1O/QfPAQ==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "2.10.0", + "@opentelemetry/core": "2.7.1", "@opentelemetry/semantic-conventions": "^1.29.0" }, "engines": { @@ -12168,22 +12088,6 @@ "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, - "node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/sdk-metrics": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.7.1.tgz", - "integrity": "sha512-MpDJdkiFDs3Pm1RHO3KByuZbuBdJEXEAkiC0+yJdsZGVCdf1RpHR6n+LHDcS7ffmfrt5kVCzJSCfm4z2C7v0uQ==", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/core": "2.7.1", - "@opentelemetry/resources": "2.7.1" - }, - "engines": { - "node": "^18.19.0 || >=20.6.0" - }, - "peerDependencies": { - "@opentelemetry/api": ">=1.9.0 <1.10.0" - } - }, "node_modules/@opentelemetry/sdk-node/node_modules/@opentelemetry/sdk-trace-base": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.7.1.tgz", diff --git a/package.json b/package.json index 66bf716c81..7f1253f473 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,7 @@ "@opentelemetry/auto-instrumentations-node": "^0.76.0", "@opentelemetry/exporter-metrics-otlp-http": "^0.218.0", "@opentelemetry/exporter-trace-otlp-http": "^0.218.0", - "@opentelemetry/sdk-metrics": "^2.7.1", + "@opentelemetry/sdk-metrics": "2.7.1", "@opentelemetry/sdk-node": "^0.218.0", "@opentelemetry/sdk-trace-base": "^2.7.1", "@railgun-community/engine": "^9.4.0", diff --git a/src/tracing.ts b/src/tracing.ts index 3a66ac279e..1b92db8432 100644 --- a/src/tracing.ts +++ b/src/tracing.ts @@ -2,6 +2,11 @@ import { SpanKind, SpanStatusCode } from '@opentelemetry/api'; import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node'; import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'; import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; +// Pinned to an exact version in package.json, not a range: the exporter and the NodeSDK below +// both depend on one exact @opentelemetry/sdk-metrics, so a range resolving higher installs a +// second copy beside theirs. The reader constructed here would then come from a different copy of +// the package than the one the SDK reads it as, and nothing in the build says so - the types are +// structural and match either way. Raise the pin together with @opentelemetry/sdk-node. import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; import { NodeSDK } from '@opentelemetry/sdk-node'; import { BatchSpanProcessor, ReadableSpan, SpanProcessor } from '@opentelemetry/sdk-trace-base'; From 4674c14fa996f878ac077b32eeff84f8df5e2698 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:36 +0200 Subject: [PATCH 62/86] Restore the line endings of two payment-link services An editing step rewrote both files with LF, which showed up as a full rewrite of files that carry three changed lines each. Also drops a local scratch file that never belonged in the tree. --- PR-BODY-4537.md | 349 ---------- .../services/payment-activation.service.ts | 628 +++++++++--------- 2 files changed, 314 insertions(+), 663 deletions(-) delete mode 100644 PR-BODY-4537.md diff --git a/PR-BODY-4537.md b/PR-BODY-4537.md deleted file mode 100644 index ece8ec3fc1..0000000000 --- a/PR-BODY-4537.md +++ /dev/null @@ -1,349 +0,0 @@ -> [!IMPORTANT] -> **`CRON_ROLE` must be set in every environment before this is deployed.** The boot aborts -> without it — deliberately, see below — so an environment missing the variable fails on the next -> deploy, whatever that deploy was for. The two repositories have separate pipelines with no -> dependency between them: merging here triggers a deploy without checking whether the variable is -> already in place. The configuration change is a separate, already open PR in the infrastructure -> repository; it is inert for the currently running image. - -> [!WARNING] -> **This PR bundles four subsystems, and a revert is all-or-nothing across all four.** That is a -> deliberate decision, recorded here so it does not surprise anyone at merge time. See -> [What this PR bundles](#what-this-pr-bundles) directly below before approving. - -## What this PR bundles - -The title says process separation, and that is the reason the branch exists. It is not all the -branch contains. Four independently revertable subsystems ended up on one branch, and because they -are one branch they are also one revert: pulling any of them out after the fact pulls out the other -three. - -| Area | What is in here | Why it is entangled | -|---|---|---| -| **Process split** | `CRON_ROLE`, `CronScope`, mandatory `scope` on 140 `@DfxCron` declarations, boot log, role heartbeat | The subject of the PR | -| **Cross-process lease** | `cron_lease` table, migration, entity, `CronLeaseService`, shutdown handler in `main.ts` | Only meaningful once two processes exist, and the split is what makes the in-process lock insufficient | -| **OpenTelemetry metrics pipeline** | `src/runtime-metrics.ts`, new dependencies, new `OTEL_METRIC_EXPORT_INTERVAL` | Written to measure the event loop saturation that motivated the split; independent of it at runtime | -| **`MonitoringService` rework** | DB-backed read path, 30 s process cache, `pessimistic_write` merge on write | Forced by the split: with the observers on the worker, the API process has no state of its own to serve | -| **`DashboardFinancialService` rework** | Aggregation stays in the worker, response building becomes its own `api` job, `LatestBalanceStore` | Forced by the split, for the same reason | - -Two of the five (metrics pipeline, and the lease insofar as it changes the schema) would stand on -their own. They are not being split out: the operator has decided against a follow-up PR, so the -bundling is stated instead of removed. What that means concretely: - -- **A revert of this PR removes the OTLP event-loop metrics** and any dashboard or alert built on - them, whether or not the reason for reverting had anything to do with them. -- **A revert re-introduces the monitoring read path that answers from process memory**, which is - wrong in a two-process deployment — so a revert here has to be accompanied by a rollback of the - role configuration, not just of this code. -- **A revert does not drop the `cron_lease` table.** The migration is not reverted by reverting the - code; the table is simply left unused. - -The safe rollback is not a revert of this PR but `CRON_ROLE=all`, under which every process -registers every job. That is the mode this branch is designed to be merged in. - -## Why - -Background jobs and HTTP requests share a single Node event loop, and Node runs JavaScript on one -thread. A CPU-heavy job therefore delays every request in the same process. - -Measured in production while investigating slow API responses: - -| Signal | Value | -|---|---| -| Event loop utilization | 84% mean, p90 100% | -| Event loop delay | mean 339 ms, p95 up to 5.3 s | -| `/version` (no DB access, 1 ms handler), measured locally against the container | p50 7 ms, **p95 5.5 s**, max 16 s | -| Actual HTTP load | 1.9 req/s | - -A p50 of a few milliseconds next to a p99 in the seconds is the signature of a blocked loop, not of -slow work: the same distribution reproduces on a route touching neither the database nor any -external service, and it persists when the network path is bypassed entirely. Database, host and -network were each ruled out separately — during one 6.1 s freeze the database had zero active -queries, and the connection pool never had a single waiting request. - -The cause is the scheduler, not traffic. Grouping the delay measurements by second-within-minute -over six hours (2,154 windows) gives stall rates of 5.8 / 52.8 / 73.3 / 81.8 / 50.0 / 21.2 % — a -factor of 14, following the scatter curve of the job start delay exactly. Request traffic in the -same grid varies by a factor of 2 and peaks elsewhere. - -## What changes - -This PR is the application half of running the same image twice: one process serving HTTP, one -running the background work. Under `CRON_ROLE=all` no existing `@DfxCron` is left out on account of -its scope — new are the role heartbeat, the Spark wallet maintenance job, `refreshLatestBalance`, -the payment delivery job, the WebSocket liveness sweep and the registration logs. - -**1. `CRON_ROLE` decides which jobs a process registers**, and `CronScope` says which process a job -belongs to. Three values each, because two would not be enough in either direction: - -- A role `all` is needed because otherwise no value covers every job — local development, the test - suite and any deployment without a separate worker would each lose something. -- A scope `api` is needed because some work is bound to the process holding the open connections — - state a request path reads in *this* process, which a job running elsewhere cannot maintain. - Delivering to those connections is a different matter and is no longer done from an `api`-scoped - job; see [The api scope and the lease](#the-api-scope-and-the-lease-pull-against-each-other) for - why that separation had to be made. - -The variable has no default and an unknown or empty value aborts the boot. Every possible default -is silent in one direction: `worker` would make a misconfigured API process run all background work -twice, `api` would make a misconfigured worker do nothing at all. - -**2. `scope` is mandatory on every `@DfxCron`.** A wrong classification fails silently — a job -wrongly scoped `worker` leaves the cache it maintains empty in the process that reads it, with no -error anywhere. A default plus an exception list moves that decision into a hand-maintained list -the compiler never sees; such a list grows and goes stale, and pinning it in a test proves the -state of the list rather than the property it stands for. - -Counted from the source tree rather than carried forward: **140 `@DfxCron` declarations — 119 -`worker`, 5 `api`, 16 `both`**, across 98 files and 34 areas; 118 carry a `process` flag, 22 do -not, five of those deliberately. 139 of the 140 have a registration path. `docs/cron-jobs.md` -carries the assignment per job, generated from the decorators, and both of its distribution tables -sum to 140. - -**3. Cases where a scope alone would not have been enough:** - -- **Monitoring state** (11 observers behind `GET /health*` and `/monitoring/data`). Scoping the - observers to the worker would freeze eleven endpoints at the boot snapshot in the API process, - the health report included; scoping them to the API process would move AML, node and bank queries - back into the request path. The state is already persisted in full, so the read path takes it from - there with a 30-second process cache. The write path needed its own answer: every process writes - the whole state as a single row, so only the metrics changed in this process are merged into the - stored row — otherwise whichever writes last drops the other's work. -- **The dashboard balance store**, written by the financial aggregation and read without touching - the database. That property was measured (23 ms median, 1,989 ms p95 before it existed) and is - kept: building the response becomes its own minute job scoped `api`, and the aggregation stays in - the worker. -- **Periodic work registered outside the scheduler** — two native `@Cron` decorators and a bare - `setInterval` driving on-chain wallet maintenance. All three would have run in both processes. - A test forbids the patterns, and carries no exceptions. - -**4. Four jobs gain a `process` flag** so a single misbehaving job can be stopped at runtime -instead of by deploy, and the worker reports itself under its own service name so its outgoing -calls do not read as API traffic. - -## Cross-process lease - -An architecture review named the load-bearing assumption of the original design: *"there is exactly -one worker, and the configuration is right"*. It was held up by convention, a runbook sentence and -an alert that **reports** a double run 15–25 minutes later. For a path that moves money, detection -is the second-best answer — and `LockClass` keeps its state in a field in process memory, so it -cannot see a second process at all. - -**Jobs scoped `worker` or `api` hold a lease in the `cron_lease` table for the duration of their -run.** A single statement decides it: the upsert takes the row over only if it has expired. Two -processes racing are serialised by the primary key and one of them gets a row back. - -**What that is worth, stated precisely.** The lease does not exclude a double run, and it does -not bound how long one lasts. A job runs once because the deployment runs one worker and because -the job tolerates being run again. What the lease adds underneath those two properties is that a -second process has to take the claim before it may **start** the job: while the holder keeps -renewing, a wrongly configured second process — a missed recreate, a second worker from `--scale`, -two processes left on `all` after a rollback — does not start it at all. If the holder stops -renewing while it is still working, the claim lapses, the second process starts, and the first runs -on to its own end; nothing here shortens that, because a running function cannot be aborted from -the outside in JavaScript and a cooperative check at every write is the same work as the fencing -token this does not carry. `CronLeaseService` says so in its own "What it does not do". - -What the lease **does** bound is the waiting: a claim left behind by a process that was killed -blocks the job for the lease TTL rather than until somebody reads an alert, and that same span is -the longest a second process waits before it may take the job over. - -**Scope `both` is deliberately exempt.** Those jobs maintain state a request path reads in *their -own* process; a lease over them would starve whichever lost the race and freeze that state. Their -safety comes from a different property: running twice has to be harmless by construction, which is -what CONTRIBUTING requires of them. - -**Why a lease and not an advisory lock.** `pg_advisory_lock` is bound to the connection and would -hold a pooled connection for the whole runtime of the job. 67 jobs declare a timeout measured in -minutes; that is a real risk to a connection pool sized by `SQL_POOL_MAX`. An expiring row -costs one short query each to take, extend and release. - -**The lease is 60 seconds, renewed every 20, and unrelated to the job's timeout.** The expiry bounds -how long a claim outlives an owner that can no longer speak for itself — SIGKILL, an OOM kill, a -lost machine. That is a property of the failure mode, not of the work, and deriving it from the -job's own timeout got it backwards: `timeout` is measured in seconds and nineteen `@DfxCron` -declarations carry 7200, so a process killed mid-run used to block its own successor from that job -for up to two hours, silently. - -**Shutdown is the other half.** Nothing in this repository ever asked for a shutdown hook, so -SIGTERM ended the process instantly and the release in the `finally` never ran. -`CronLeaseService.shutdown` now waits up to ten seconds for the runs this process holds, so their -normal release hands the job to the successor. A run still going after that **keeps** its lease: -taking it away would let the successor start the same job while this process works on it for the -rest of the grace, which is the outcome the lease exists to make rare. - -It hangs off a SIGTERM/SIGINT handler rather than `app.enableShutdownHooks()`, and that is -deliberate. The idiomatic call is global: it would also start running the nine `onModuleDestroy` -implementations this application carries but has never executed, and Nest runs those *before* the -lease hook — they empty the strategy registries that PayIn, PayOut and DEX jobs resolve from. On -its own that is harmless, since the process was about to die. Next to this change it is not: the -wait deliberately keeps in-flight jobs alive longer, so a running payout would gain time to fail on -an emptied registry rather than simply be cut off. A test pins that the idiomatic call stays out. - -**An unusable lease table is reported rather than silent.** Without it every worker- and api-scoped -job is skipped on every tick — the right behaviour, and what CONTRIBUTING asks for, but the skip -used to look exactly like a job with nothing to do. The role heartbeat is scope `both` and -therefore exempt from the lease, so it kept reporting a healthy process while everything it counts -sat out; and it counts *registered* jobs, which cannot see this at all. The lease layer now reads -the table at start-up and carries a health flag that stays false until an operation gets through; -the heartbeat writes the same line at error level with the reason appended when it is bad. The same -line, because the role alert matches on its shape. - -**What it still cannot do — and the code says so.** If the holder stops renewing while a job runs, -the claim expires and a second process can start the job while the first is still working, for as -long as that first run takes. Full fencing would need a token on every single write. What the lease -shortens is the wait, not the overlap. - -**With the database unreachable the job does not run.** A job that moves money must not proceed on -the assumption that it is probably alone — and that is exactly when the assumption is least safe. - -### The api scope and the lease pull against each other - -`PaymentCronService` writes to the database and triggers merchant webhooks, which argues for one -process, and it used to be the only thing releasing the callers waiting on the process that ran it, -which argues for every process. A single scope cannot satisfy both, and the lease made that -visible: whichever process lost the claim left its callers waiting for nothing. - -The resolution taken is to split the job rather than the scope. The writing runs under the lease -(`Worker`), and `deliverPaymentUpdates` delivers from the state those writes leave behind, scoped -`Both`, in every process, without a lease — it writes nothing and calls nothing outside its own -process, which is what allows it to run everywhere. An `api`-scoped job losing the race is still -logged at error level, unlike a worker job, which loses it every cycle by design. - -Leaving `Pending` is decided by the update statement rather than by a status read, because the -expiry timers stay in the process that served the request while the expiry job runs in the worker, -and cancelling and completing arrive from request paths as well. Only the caller the database lets -past `Pending` sends the merchant its webhook. - -## Schema - -`migration/1785600000000-AddCronLease.js` creates `cron_lease`. Two things worth stating: - -- The primary key is `PK_a12c181c2b26f33be13d55a15af` — `PK_` plus the first 27 characters of - `sha1('cron_lease_name')`, which is what TypeORM's own naming strategy produces. A hand-picked - name would not be recognised by a schema comparison, which would then offer to create the - constraint. A new test recomputes every primary key declared in a `CREATE TABLE` across all - migrations (102 of them, all matching) and rejects any spelled-out constraint name. -- `acquired` and `expires` are `timestamptz`. They are compared against `now()` in raw SQL, and a - value without a zone on one side of that comparison resolves through whatever time zone the - session carries: the same row expires an hour late or an hour early across a daylight saving - change. An hour late is a job that runs nowhere, an hour early is two processes running it. - -`src/shared/models/cron-lease/cron-lease.entity.ts` mirrors the table. The service never reads -through a repository — the claim is a single `INSERT .. ON CONFLICT .. WHERE` the query builder -cannot express — but a table that exists only as DDL is invisible to the entity model, and the next -generated migration would read that absence as an instruction to `DROP TABLE "cron_lease"`. A test -builds the entity metadata without a connection and checks it against the migration file. - -## Deployment - -The order matters and is not optional: - -1. **`CRON_ROLE=all` must be set in every environment before this PR is deployed.** The boot aborts - without it, so an environment missing the variable fails on the next routine deploy — for the - currently running image the variable is unknown and therefore inert. -2. Merge and deploy this PR. The migration creating `cron_lease` runs with it. -3. Observe: the boot log states the split — `CronRole all: registered 139 of 139 jobs (worker: 119, - api: 4, both: 16)` — and health and dashboard endpoints answer unchanged. -4. Alerting, log-level normalisation, runbooks and dashboards — before, not after the next step. -5. Create the second process and set the roles. With the roles split, the boot log reads - `registered 135 of 139` in the worker and `registered 20 of 139` in the HTTP process. - -Rolling back never means reverting this PR: under `CRON_ROLE=all` its content is today's behaviour, -so what gets reset is configuration. See [What this PR bundles](#what-this-pr-bundles) for why a -revert is the expensive option. - -## Testing - -`npm test`, plus targeted runs on the affected suites; `tsc --noEmit`, ESLint and Prettier clean. -Coverage added by this branch: - -- every rejected value for the role, including the empty string and the absent variable, asserting - a throw rather than a silent default -- the `env -> Config` wiring the cron service actually reads, not just the parser -- registration per role: `all` registers everything, each role drops the other's scope, `both` - survives in all three -- which jobs pass through the lease, and that a job declaring `timeout: 7200` still claims for 60 - seconds — the regression that made a deployment block a job for two hours -- shutdown: the lease survives a shutdown that outlasts the grace period, the shutdown does not - return before the job does, `main.ts` is pinned to wire it to the signal at all, and pinned NOT - to reach for `app.enableShutdownHooks()` -- an unusable lease table: reported at start-up, still reported on the next heartbeat when no new - failure occurred, and reported healthy again once a claim gets through -- an api-scoped job losing the race is reported; a worker job losing it is not -- constraint naming across every migration, and the `cron_lease` entity against its own DDL -- the monitoring service: read path answers from the persisted state including the filtered - queries, the merge prefers whichever value is newer, the write path keeps metrics another process - wrote, the row is read under a write lock in the same transaction it writes in, an older - measurement is not put back over a newer one, and the merge is retried only on errors a retry can - resolve -- the monitoring state row: an environment whose state does not live under `id: 1` is answered from - the row that exists, and the write seeds `id: 1` from it rather than with a partial state -- the statistic start-up fill: not in the worker, yes in `api` and `all`, off when the process flag - is off, and a failure reported instead of left as an unhandled rejection -- the Spark wallet maintenance: registered as a job rather than a timer, scoped `worker` -- the guard against `@Cron(`, `@Interval(`, `@Timeout(` and `setInterval(`, including a check that - its exception list still matches something - -Every fix in the review rounds below was verified by putting the defect back and watching the test -fail, then restoring it. - -## Known discrepancies, recorded rather than fixed - -`ExchangeController::checkTrades` is **never registered**: its class is listed under `controllers:` -in `ExchangeModule` and nowhere under `providers:`, and `DiscoveryService.getProviders()` does not -return controllers, so the scan never sees the decorator. This predates the process split — the job -has never run. `TransactionController::checkLists` looks like the same case but is not: -`HistoryModule` lists that class under both `controllers:` and `providers:`, so the job is -registered, on the provider instance, which is a different object from the controller instance the -request handlers use. - -`CitreaBaseStrategy::checkPayInEntries` is declared on an **abstract** class, so it is registered -once per concrete subclass rather than once per declaration. There is currently one subclass. - -16 jobs carry no `process` flag. That is pre-existing, named in `docs/cron-jobs.md` as an omission -rather than hidden, and retrofitting it means introducing 16 new kill switches — a decision about -those jobs, not about this PR. - -What should happen to any of these is a decision about the jobs, not about this inventory. - -## Review rounds - -**Round 1 — rebased** onto the current `develop`, conflict-free, CI 12/12 green. - -**Round 2 — role heartbeat added.** `DfxCronService::reportRole` writes -`CronRole : heartbeat, N jobs registered` in every process every ten minutes. The reason lies -outside this repository but the line originates here: the Grafana rule meant to report a wrong role -assignment used to read the boot line, which is written exactly once. On a healthy system a -counting window over it reports permanently from the day after the last deploy, because the line -falls out of the window while the container keeps running. And the most expensive state is -precisely the one without a restart: if the recreate for a configuration change does not happen, -the HTTP process keeps its old role while the worker takes over the same jobs. Without a restart -there is no new boot line, so the rule could not see it structurally. Scope `both` so the line -appears in every process, with the role *in* the line; no `process` flag, because a watchdog that -can be switched off looks, switched off, exactly like the failure it reports. `useDelay: false`, -because the alert reads a 12-minute window and the jitter is adjustable from outside through -`CRON_JOB_DELAY` — a watchdog must not have its timing tuned by a knob meant for spreading load. - -**Round 3 — seven review points**, three implemented, four re-measured and declined with the -measurement stated. Implemented: the heartbeat tests were reading decorator metadata and would have -stayed green if the scan never saw the method, so the test now hands the scan its own service -instance the way Nest does and expects the heartbeat in the count; the guard test's `setTimeout` -gap was documented with its evidence; and the claim "behaviourally identical to today under -`CRON_ROLE=all`" was narrowed to what is actually true. Declined: the `git diff --check` whitespace -report (all files are CRLF and `.prettierrc` sets `endOfLine: auto`; two consecutive `develop` -commits report the same), the `PaymentCronService` scope (see above — the concern was right and is -now addressed by the lease and by reporting the lost race), the 16 missing flags, and the -"environment updates missing" point (they are in the infrastructure repository, because that is -where `CRON_ROLE` is set). - -**Round 4 — the cross-process lease**, described above. - -**Round 5 — a five-instance review of this PR and its three infrastructure counterparts.** Nine -findings here, all fixed on this branch: the lease TTL derived from the job timeout (up to a -two-hour outage per deployment) and the missing shutdown path; an unusable lease table looking like -a healthy process; a primary key name that violates the deterministic-naming rule; the missing -entity; timestamps without a time zone; the Spark maintenance timer and the statistic start-up fill -both running outside the scheduler and therefore outside the lease; the `api` scope contradicting -the lease; the monitoring read path depending on a row with `id: 1`; and this bundling, which is -named here rather than split out. diff --git a/src/subdomains/core/payment-link/services/payment-activation.service.ts b/src/subdomains/core/payment-link/services/payment-activation.service.ts index 06ccb12137..4a98268874 100644 --- a/src/subdomains/core/payment-link/services/payment-activation.service.ts +++ b/src/subdomains/core/payment-link/services/payment-activation.service.ts @@ -1,314 +1,314 @@ -import { BadRequestException, ConflictException, Injectable, NotFoundException } from '@nestjs/common'; -import { Config } from 'src/config/config'; -import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.enum'; -import { CryptoService } from 'src/integration/blockchain/shared/services/crypto.service'; -import { LnBitsWalletPaymentParamsDto } from 'src/integration/lightning/dto/lnbits.dto'; -import { LightningClient } from 'src/integration/lightning/lightning-client'; -import { LightningHelper } from 'src/integration/lightning/lightning-helper'; -import { LightningService } from 'src/integration/lightning/services/lightning.service'; -import { Asset } from 'src/shared/models/asset/asset.entity'; -import { AssetService } from 'src/shared/models/asset/asset.service'; -import { DfxLogger } from 'src/shared/services/dfx-logger'; -import { Util } from 'src/shared/utils/util'; -import { C2BPaymentLinkService } from 'src/subdomains/core/payment-link/services/c2b-payment-link.service'; -import { EntityManager, Equal, LessThan, Not, Repository } from 'typeorm'; -import { TransferInfo } from '../dto/payment-link.dto'; -import { PaymentActivation } from '../entities/payment-activation.entity'; -import { PaymentLinkPayment } from '../entities/payment-link-payment.entity'; -import { PaymentQuote } from '../entities/payment-quote.entity'; -import { PaymentActivationStatus, PaymentLinkPaymentMode, PaymentStandard } from '../enums'; -import { PaymentActivationRepository } from '../repositories/payment-activation.repository'; -import { PaymentBalanceService } from './payment-balance.service'; -import { PaymentQuoteService } from './payment-quote.service'; - -@Injectable() -export class PaymentActivationService { - private readonly logger = new DfxLogger(PaymentActivationService); - - private readonly client: LightningClient; - - constructor( - readonly lightningService: LightningService, - private readonly paymentActivationRepo: PaymentActivationRepository, - private readonly paymentQuoteService: PaymentQuoteService, - private readonly paymentBalanceService: PaymentBalanceService, - private readonly assetService: AssetService, - private readonly cryptoService: CryptoService, - private readonly c2bPaymentLinkService: C2BPaymentLinkService, - ) { - this.client = lightningService.getDefaultClient(); - } - - async close(activation: PaymentActivation): Promise { - await this.paymentActivationRepo.update( - { id: activation.id, status: Not(PaymentActivationStatus.CLOSED) }, - { status: PaymentActivationStatus.CLOSED }, - ); - } - - /** `manager` runs the closes in the caller's transaction, see `cancelAllForPayment` for why. */ - async closeAllForPayment(paymentId: number, manager?: EntityManager): Promise { - const repo: Repository = manager?.getRepository(PaymentActivation) ?? this.paymentActivationRepo; - - await repo.update( - { payment: { id: paymentId }, status: Not(PaymentActivationStatus.CLOSED) }, - { status: PaymentActivationStatus.CLOSED }, - ); - } - - async closeAllForQuote(quoteId: number): Promise { - await this.paymentActivationRepo.update( - { quote: { id: quoteId }, status: Not(PaymentActivationStatus.CLOSED) }, - { status: PaymentActivationStatus.CLOSED }, - ); - } - - async getActivationByTxId(txHash: string): Promise { - return this.paymentActivationRepo.findOne({ - where: { paymentHash: Equal(txHash), status: PaymentActivationStatus.OPEN }, - relations: { quote: { payment: true } }, - }); - } - - async deleteActivation(activation: PaymentActivation): Promise { - await this.paymentActivationRepo.delete(activation.id); - } - - // --- HANDLE PENDING ACTIVATIONS --- // - async processExpiredActivations(): Promise { - const maxDate = Util.secondsBefore(Config.payment.timeoutDelay); - - await this.paymentActivationRepo.update( - { - status: PaymentActivationStatus.OPEN, - expiryDate: LessThan(maxDate), - }, - { status: PaymentActivationStatus.CLOSED }, - ); - } - - // --- CREATE ACTIVATIONS --- // - - async doCreateRequest(pendingPayment: PaymentLinkPayment, transferInfo: TransferInfo): Promise { - const actualQuote = await this.paymentQuoteService.getActualQuote(pendingPayment, transferInfo); - if (!actualQuote) throw new NotFoundException(`No matching actual quote found`); - - if (transferInfo.quoteUniqueId) { - const transferAmount = actualQuote.getTransferAmountFor(transferInfo.method, transferInfo.asset)?.amount; - - if (!transferAmount) throw new BadRequestException(`Invalid method or asset`); - - transferInfo.amount = transferAmount; - } - - const expiryDate = new Date(Math.min(pendingPayment.expiryDate.getTime(), actualQuote.expiryDate.getTime())); - - const expirySec = Util.secondsDiff(new Date(), expiryDate); - if (expirySec < 1) throw new BadRequestException('Quote is expired'); - - const activations = await this.getExistingActivations(transferInfo); - if ( - actualQuote.standard === PaymentStandard.PAY_TO_ADDRESS && - activations.some( - (a) => - a.standard === PaymentStandard.PAY_TO_ADDRESS && - (a.quote.id !== actualQuote.id || pendingPayment.mode === PaymentLinkPaymentMode.MULTIPLE), - ) - ) - throw new ConflictException('Duplicate payment request'); - - return ( - activations.find((a) => a.quote.id === actualQuote.id) ?? - this.createNewPaymentActivationRequest( - pendingPayment, - actualQuote, - transferInfo, - expirySec, - expiryDate, - actualQuote.standard, - ) - ); - } - - private async getExistingActivations(transferInfo: TransferInfo): Promise { - return this.paymentActivationRepo.find({ - where: { - status: PaymentActivationStatus.OPEN, - amount: transferInfo.amount, - method: transferInfo.method, - asset: { uniqueName: `${transferInfo.method}/${transferInfo.asset}` }, - }, - relations: { - payment: true, - quote: true, - }, - }); - } - - private async createNewPaymentActivationRequest( - payment: PaymentLinkPayment, - quote: PaymentQuote, - transferInfo: TransferInfo, - expirySec: number, - expiryDate: Date, - standard: PaymentStandard, - ): Promise { - const { paymentRequest, paymentHash } = await this.createBlockchainRequest(payment, transferInfo, expirySec, quote); - - return this.savePaymentActivationRequest( - payment, - quote, - paymentRequest, - paymentHash, - transferInfo, - expiryDate, - standard, - ); - } - - private async createBlockchainRequest( - payment: PaymentLinkPayment, - transferInfo: TransferInfo, - expirySec: number, - quote: PaymentQuote, - ): Promise<{ paymentRequest: string; paymentHash?: string }> { - switch (transferInfo.method) { - case Blockchain.LIGHTNING: - return this.createLightningRequest(payment, transferInfo, expirySec); - - case Blockchain.BITCOIN: - case Blockchain.FIRO: - case Blockchain.MONERO: - case Blockchain.ZANO: - case Blockchain.ETHEREUM: - case Blockchain.SEPOLIA: - case Blockchain.ARBITRUM: - case Blockchain.OPTIMISM: - case Blockchain.BASE: - case Blockchain.GNOSIS: - case Blockchain.POLYGON: - case Blockchain.BINANCE_SMART_CHAIN: - case Blockchain.SOLANA: - case Blockchain.TRON: - case Blockchain.CARDANO: - case Blockchain.INTERNET_COMPUTER: { - const address = this.paymentBalanceService.getDepositAddress(transferInfo.method); - if (address) return this.createPaymentRequest(address, transferInfo, 'DFX Payment'); - - break; - } - - case Blockchain.KUCOIN_PAY: - case Blockchain.BINANCE_PAY: - return this.createC2BPaymentRequest(payment, transferInfo, quote); - } - - throw new BadRequestException(`Invalid method ${transferInfo.method}`); - } - - private async createLightningRequest( - payment: PaymentLinkPayment, - transferInfo: TransferInfo, - expirySec: number, - ): Promise<{ paymentRequest: string; paymentHash: string }> { - const lnurlpAddress = await this.getDepositLnurlpAddress(payment); - if (!lnurlpAddress) throw new BadRequestException('Deposit LNURLp Address not found'); - - const uniqueId = payment.uniqueId; - const uniqueIdSignature = Util.createSign(uniqueId, Config.blockchain.lightning.lnbits.signingPrivKey); - - const walletPaymentParams: LnBitsWalletPaymentParamsDto = { - amount: LightningHelper.btcToSat(transferInfo.amount), - memo: payment.memo, - expirySec: expirySec, - webhook: `${Config.url()}/payIn/lnurlpPayment/${uniqueId}`, - extra: { - link: lnurlpAddress, - signature: uniqueIdSignature, - }, - }; - - const paymentRequest = await this.client.getLnBitsWalletPayment(walletPaymentParams).then((r) => r.pr); - const paymentHash = LightningHelper.getPaymentHashOfInvoice(paymentRequest); - - return { paymentRequest, paymentHash }; - } - - private async getDepositLnurlpAddress(pendingPayment: PaymentLinkPayment): Promise { - try { - const depositAddress = pendingPayment.link.route.deposit.address; - - if (!depositAddress.startsWith('LNURL')) { - this.logger.error( - `Lightning transaction: Deposit address ${depositAddress} is not a LNURL address for payment link ${pendingPayment.link.uniqueId}`, - ); - return; - } - - const decodedDepositAddress = LightningHelper.decodeLnurl(depositAddress); - const paths = decodedDepositAddress.split('/'); - return paths[paths.length - 1]; - } catch (e) { - this.logger.error( - `Lightning transaction: Cannot get LNURLp address for payment link ${pendingPayment.link.id}`, - e, - ); - } - } - - private async createPaymentRequest( - address: string, - transferInfo: TransferInfo, - label?: string, - ): Promise<{ paymentRequest: string; paymentHash?: string }> { - const asset = await this.getAssetByInfo(transferInfo); - - const paymentRequest = await this.cryptoService.getPaymentRequest(true, asset, address, transferInfo.amount, label); - return { paymentRequest }; - } - - private async createC2BPaymentRequest( - payment: PaymentLinkPayment, - transferInfo: TransferInfo, - quote: PaymentQuote, - ): Promise<{ paymentRequest: string; paymentHash: string }> { - const order = await this.c2bPaymentLinkService.createOrder(payment, transferInfo, quote); - return { paymentRequest: order.paymentRequest, paymentHash: order.providerOrderId }; - } - - private async savePaymentActivationRequest( - payment: PaymentLinkPayment, - quote: PaymentQuote, - paymentRequest: string, - paymentHash: string, - transferInfo: TransferInfo, - expiryDate: Date, - standard: PaymentStandard, - ): Promise { - const asset = await this.getAssetByInfo(transferInfo); - - const newPaymentActivation = this.paymentActivationRepo.create({ - status: PaymentActivationStatus.OPEN, - method: transferInfo.method, - amount: transferInfo.amount, - asset, - paymentRequest, - paymentHash, - expiryDate, - standard, - payment, - quote, - }); - - return this.paymentActivationRepo.save(newPaymentActivation); - } - - private async getAssetByInfo(transferInfo: TransferInfo): Promise { - const uniqueName = `${transferInfo.method}/${transferInfo.asset}`; - - const asset = await this.assetService.getAssetByUniqueName(uniqueName); - if (!asset) throw new NotFoundException(`Asset ${uniqueName} not found`); - - return asset; - } -} +import { BadRequestException, ConflictException, Injectable, NotFoundException } from '@nestjs/common'; +import { Config } from 'src/config/config'; +import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.enum'; +import { CryptoService } from 'src/integration/blockchain/shared/services/crypto.service'; +import { LnBitsWalletPaymentParamsDto } from 'src/integration/lightning/dto/lnbits.dto'; +import { LightningClient } from 'src/integration/lightning/lightning-client'; +import { LightningHelper } from 'src/integration/lightning/lightning-helper'; +import { LightningService } from 'src/integration/lightning/services/lightning.service'; +import { Asset } from 'src/shared/models/asset/asset.entity'; +import { AssetService } from 'src/shared/models/asset/asset.service'; +import { DfxLogger } from 'src/shared/services/dfx-logger'; +import { Util } from 'src/shared/utils/util'; +import { C2BPaymentLinkService } from 'src/subdomains/core/payment-link/services/c2b-payment-link.service'; +import { EntityManager, Equal, LessThan, Not, Repository } from 'typeorm'; +import { TransferInfo } from '../dto/payment-link.dto'; +import { PaymentActivation } from '../entities/payment-activation.entity'; +import { PaymentLinkPayment } from '../entities/payment-link-payment.entity'; +import { PaymentQuote } from '../entities/payment-quote.entity'; +import { PaymentActivationStatus, PaymentLinkPaymentMode, PaymentStandard } from '../enums'; +import { PaymentActivationRepository } from '../repositories/payment-activation.repository'; +import { PaymentBalanceService } from './payment-balance.service'; +import { PaymentQuoteService } from './payment-quote.service'; + +@Injectable() +export class PaymentActivationService { + private readonly logger = new DfxLogger(PaymentActivationService); + + private readonly client: LightningClient; + + constructor( + readonly lightningService: LightningService, + private readonly paymentActivationRepo: PaymentActivationRepository, + private readonly paymentQuoteService: PaymentQuoteService, + private readonly paymentBalanceService: PaymentBalanceService, + private readonly assetService: AssetService, + private readonly cryptoService: CryptoService, + private readonly c2bPaymentLinkService: C2BPaymentLinkService, + ) { + this.client = lightningService.getDefaultClient(); + } + + async close(activation: PaymentActivation): Promise { + await this.paymentActivationRepo.update( + { id: activation.id, status: Not(PaymentActivationStatus.CLOSED) }, + { status: PaymentActivationStatus.CLOSED }, + ); + } + + /** `manager` runs the closes in the caller's transaction, see `cancelAllForPayment` for why. */ + async closeAllForPayment(paymentId: number, manager?: EntityManager): Promise { + const repo: Repository = manager?.getRepository(PaymentActivation) ?? this.paymentActivationRepo; + + await repo.update( + { payment: { id: paymentId }, status: Not(PaymentActivationStatus.CLOSED) }, + { status: PaymentActivationStatus.CLOSED }, + ); + } + + async closeAllForQuote(quoteId: number): Promise { + await this.paymentActivationRepo.update( + { quote: { id: quoteId }, status: Not(PaymentActivationStatus.CLOSED) }, + { status: PaymentActivationStatus.CLOSED }, + ); + } + + async getActivationByTxId(txHash: string): Promise { + return this.paymentActivationRepo.findOne({ + where: { paymentHash: Equal(txHash), status: PaymentActivationStatus.OPEN }, + relations: { quote: { payment: true } }, + }); + } + + async deleteActivation(activation: PaymentActivation): Promise { + await this.paymentActivationRepo.delete(activation.id); + } + + // --- HANDLE PENDING ACTIVATIONS --- // + async processExpiredActivations(): Promise { + const maxDate = Util.secondsBefore(Config.payment.timeoutDelay); + + await this.paymentActivationRepo.update( + { + status: PaymentActivationStatus.OPEN, + expiryDate: LessThan(maxDate), + }, + { status: PaymentActivationStatus.CLOSED }, + ); + } + + // --- CREATE ACTIVATIONS --- // + + async doCreateRequest(pendingPayment: PaymentLinkPayment, transferInfo: TransferInfo): Promise { + const actualQuote = await this.paymentQuoteService.getActualQuote(pendingPayment, transferInfo); + if (!actualQuote) throw new NotFoundException(`No matching actual quote found`); + + if (transferInfo.quoteUniqueId) { + const transferAmount = actualQuote.getTransferAmountFor(transferInfo.method, transferInfo.asset)?.amount; + + if (!transferAmount) throw new BadRequestException(`Invalid method or asset`); + + transferInfo.amount = transferAmount; + } + + const expiryDate = new Date(Math.min(pendingPayment.expiryDate.getTime(), actualQuote.expiryDate.getTime())); + + const expirySec = Util.secondsDiff(new Date(), expiryDate); + if (expirySec < 1) throw new BadRequestException('Quote is expired'); + + const activations = await this.getExistingActivations(transferInfo); + if ( + actualQuote.standard === PaymentStandard.PAY_TO_ADDRESS && + activations.some( + (a) => + a.standard === PaymentStandard.PAY_TO_ADDRESS && + (a.quote.id !== actualQuote.id || pendingPayment.mode === PaymentLinkPaymentMode.MULTIPLE), + ) + ) + throw new ConflictException('Duplicate payment request'); + + return ( + activations.find((a) => a.quote.id === actualQuote.id) ?? + this.createNewPaymentActivationRequest( + pendingPayment, + actualQuote, + transferInfo, + expirySec, + expiryDate, + actualQuote.standard, + ) + ); + } + + private async getExistingActivations(transferInfo: TransferInfo): Promise { + return this.paymentActivationRepo.find({ + where: { + status: PaymentActivationStatus.OPEN, + amount: transferInfo.amount, + method: transferInfo.method, + asset: { uniqueName: `${transferInfo.method}/${transferInfo.asset}` }, + }, + relations: { + payment: true, + quote: true, + }, + }); + } + + private async createNewPaymentActivationRequest( + payment: PaymentLinkPayment, + quote: PaymentQuote, + transferInfo: TransferInfo, + expirySec: number, + expiryDate: Date, + standard: PaymentStandard, + ): Promise { + const { paymentRequest, paymentHash } = await this.createBlockchainRequest(payment, transferInfo, expirySec, quote); + + return this.savePaymentActivationRequest( + payment, + quote, + paymentRequest, + paymentHash, + transferInfo, + expiryDate, + standard, + ); + } + + private async createBlockchainRequest( + payment: PaymentLinkPayment, + transferInfo: TransferInfo, + expirySec: number, + quote: PaymentQuote, + ): Promise<{ paymentRequest: string; paymentHash?: string }> { + switch (transferInfo.method) { + case Blockchain.LIGHTNING: + return this.createLightningRequest(payment, transferInfo, expirySec); + + case Blockchain.BITCOIN: + case Blockchain.FIRO: + case Blockchain.MONERO: + case Blockchain.ZANO: + case Blockchain.ETHEREUM: + case Blockchain.SEPOLIA: + case Blockchain.ARBITRUM: + case Blockchain.OPTIMISM: + case Blockchain.BASE: + case Blockchain.GNOSIS: + case Blockchain.POLYGON: + case Blockchain.BINANCE_SMART_CHAIN: + case Blockchain.SOLANA: + case Blockchain.TRON: + case Blockchain.CARDANO: + case Blockchain.INTERNET_COMPUTER: { + const address = this.paymentBalanceService.getDepositAddress(transferInfo.method); + if (address) return this.createPaymentRequest(address, transferInfo, 'DFX Payment'); + + break; + } + + case Blockchain.KUCOIN_PAY: + case Blockchain.BINANCE_PAY: + return this.createC2BPaymentRequest(payment, transferInfo, quote); + } + + throw new BadRequestException(`Invalid method ${transferInfo.method}`); + } + + private async createLightningRequest( + payment: PaymentLinkPayment, + transferInfo: TransferInfo, + expirySec: number, + ): Promise<{ paymentRequest: string; paymentHash: string }> { + const lnurlpAddress = await this.getDepositLnurlpAddress(payment); + if (!lnurlpAddress) throw new BadRequestException('Deposit LNURLp Address not found'); + + const uniqueId = payment.uniqueId; + const uniqueIdSignature = Util.createSign(uniqueId, Config.blockchain.lightning.lnbits.signingPrivKey); + + const walletPaymentParams: LnBitsWalletPaymentParamsDto = { + amount: LightningHelper.btcToSat(transferInfo.amount), + memo: payment.memo, + expirySec: expirySec, + webhook: `${Config.url()}/payIn/lnurlpPayment/${uniqueId}`, + extra: { + link: lnurlpAddress, + signature: uniqueIdSignature, + }, + }; + + const paymentRequest = await this.client.getLnBitsWalletPayment(walletPaymentParams).then((r) => r.pr); + const paymentHash = LightningHelper.getPaymentHashOfInvoice(paymentRequest); + + return { paymentRequest, paymentHash }; + } + + private async getDepositLnurlpAddress(pendingPayment: PaymentLinkPayment): Promise { + try { + const depositAddress = pendingPayment.link.route.deposit.address; + + if (!depositAddress.startsWith('LNURL')) { + this.logger.error( + `Lightning transaction: Deposit address ${depositAddress} is not a LNURL address for payment link ${pendingPayment.link.uniqueId}`, + ); + return; + } + + const decodedDepositAddress = LightningHelper.decodeLnurl(depositAddress); + const paths = decodedDepositAddress.split('/'); + return paths[paths.length - 1]; + } catch (e) { + this.logger.error( + `Lightning transaction: Cannot get LNURLp address for payment link ${pendingPayment.link.id}`, + e, + ); + } + } + + private async createPaymentRequest( + address: string, + transferInfo: TransferInfo, + label?: string, + ): Promise<{ paymentRequest: string; paymentHash?: string }> { + const asset = await this.getAssetByInfo(transferInfo); + + const paymentRequest = await this.cryptoService.getPaymentRequest(true, asset, address, transferInfo.amount, label); + return { paymentRequest }; + } + + private async createC2BPaymentRequest( + payment: PaymentLinkPayment, + transferInfo: TransferInfo, + quote: PaymentQuote, + ): Promise<{ paymentRequest: string; paymentHash: string }> { + const order = await this.c2bPaymentLinkService.createOrder(payment, transferInfo, quote); + return { paymentRequest: order.paymentRequest, paymentHash: order.providerOrderId }; + } + + private async savePaymentActivationRequest( + payment: PaymentLinkPayment, + quote: PaymentQuote, + paymentRequest: string, + paymentHash: string, + transferInfo: TransferInfo, + expiryDate: Date, + standard: PaymentStandard, + ): Promise { + const asset = await this.getAssetByInfo(transferInfo); + + const newPaymentActivation = this.paymentActivationRepo.create({ + status: PaymentActivationStatus.OPEN, + method: transferInfo.method, + amount: transferInfo.amount, + asset, + paymentRequest, + paymentHash, + expiryDate, + standard, + payment, + quote, + }); + + return this.paymentActivationRepo.save(newPaymentActivation); + } + + private async getAssetByInfo(transferInfo: TransferInfo): Promise { + const uniqueName = `${transferInfo.method}/${transferInfo.asset}`; + + const asset = await this.assetService.getAssetByUniqueName(uniqueName); + if (!asset) throw new NotFoundException(`Asset ${uniqueName} not found`); + + return asset; + } +} From 1e7f8d26f330e84e0483d4eeeca0b238a3ce7979 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:37 +0200 Subject: [PATCH 63/86] Let the latest-balance read fill its own store The store said there was one writer per process, but the job filling it is leased: with more than one API process, one takes the tick and the rest never run it, so their store stayed at whatever they started with. It now loads on demand through AsyncCache and ages out after a minute, which is what CONTRIBUTING asks of a cache a request path reads; the job is refresh only, and the start-up fill it replaced is gone. --- .../dashboard-financial.service.spec.ts | 166 ++++++++---------- .../dashboard/dashboard-financial.service.ts | 78 +++----- .../dashboard/latest-balance.store.ts | 39 ++-- 3 files changed, 126 insertions(+), 157 deletions(-) diff --git a/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts b/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts index dfe80ec540..e0bbc0c1df 100644 --- a/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts +++ b/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts @@ -10,8 +10,6 @@ import { FinancialLogSummary } from '../../log/log.repository'; import { LogService } from '../../log/log.service'; import { DashboardFinancialService } from '../dashboard-financial.service'; import { LatestBalanceResponseDto } from '../dto/financial-log.dto'; -import * as ProcessService from 'src/shared/services/process.service'; -import { Config, CronRole } from 'src/config/config'; import { TestUtil } from 'src/shared/utils/test.util'; import { LatestBalanceStore } from '../latest-balance.store'; @@ -19,22 +17,20 @@ describe('DashboardFinancialService', () => { let service: DashboardFinancialService; let logService: LogService; let assetService: AssetService; + /** + * The real store, not a double: what is under test here is that a request fills it, so a double + * answering `get` with whatever it was told would test the opposite of the point. + */ let latestBalanceStore: LatestBalanceStore; - // Config is only populated once TestUtil.provideConfig has built it, so the original role is - // captured after the module is compiled, not while the suite is being defined. - let originalRole: CronRole; - afterEach(() => { - if (originalRole !== undefined) Config.cronRole = originalRole; - jest.restoreAllMocks(); }); beforeEach(async () => { logService = createMock(); assetService = createMock(); - latestBalanceStore = createMock(); + latestBalanceStore = new LatestBalanceStore(); const module: TestingModule = await Test.createTestingModule({ providers: [ @@ -48,11 +44,6 @@ describe('DashboardFinancialService', () => { }).compile(); service = module.get(DashboardFinancialService); - originalRole ??= Config.cronRole; - - // DisabledProcess fails closed when the process flags were never loaded, which is the state of - // a bare testing module. The tests that care about the flag set it themselves. - jest.spyOn(ProcessService, 'DisabledProcess').mockReturnValue(false); }); function logEntry(): Log { @@ -63,7 +54,7 @@ describe('DashboardFinancialService', () => { } as Log; } - /** Mocks the log entry and the assets the job resolves, then runs it. */ + /** Mocks the log entry and the assets the aggregation resolves, then runs the refresh job. */ async function refreshFrom( timestamp: Date, assetLog: AssetLog, @@ -351,28 +342,44 @@ describe('DashboardFinancialService', () => { }); }); - describe('getLatestBalance (write-through store read)', () => { - it('returns undefined when the store is empty and never touches the database', async () => { - jest.spyOn(latestBalanceStore, 'get').mockReturnValue(undefined); - const getLatestFinancialLogSpy = jest.spyOn(logService, 'getLatestFinancialLog'); - const getAssetsByIdSpy = jest.spyOn(assetService, 'getAssetsById'); - - const result = await service.getLatestBalance(); + describe('getLatestBalance (cached read that loads itself)', () => { + it('returns undefined when the database holds no financial log at all', async () => { + jest.spyOn(logService, 'getLatestFinancialLog').mockResolvedValue(undefined); - expect(result).toBeUndefined(); - expect(getLatestFinancialLogSpy).toHaveBeenCalledTimes(0); - expect(getAssetsByIdSpy).toHaveBeenCalledTimes(0); + await expect(service.getLatestBalance()).resolves.toBeUndefined(); }); - it('returns exactly the value held in the store (pure store read)', async () => { - const cached: LatestBalanceResponseDto = { + it('loads the aggregate itself when no job has filled the store in this process', async () => { + // The refresh job is leased: with several API processes it runs in one of them per tick, so + // a request served anywhere else has to be able to fill the store on its own. + jest.spyOn(logService, 'getLatestFinancialLog').mockResolvedValue(logEntry()); + + await expect(service.getLatestBalance()).resolves.toEqual({ timestamp: new Date('2026-07-14T12:00:00Z'), - byType: [{ name: 'Crypto', plusBalanceChf: 100, minusBalanceChf: 0, netBalanceChf: 100 }], - byBlockchain: [{ name: 'Ethereum', plusBalanceChf: 100, minusBalanceChf: 0, netBalanceChf: 100 }], - }; - jest.spyOn(latestBalanceStore, 'get').mockReturnValue(cached); + byType: [], + byBlockchain: [], + }); + }); - await expect(service.getLatestBalance()).resolves.toBe(cached); + it('does not read the database again while the entry it holds is current', async () => { + const getLatestFinancialLogSpy = jest.spyOn(logService, 'getLatestFinancialLog').mockResolvedValue(logEntry()); + + await service.getLatestBalance(); + await service.getLatestBalance(); + + expect(getLatestFinancialLogSpy).toHaveBeenCalledTimes(1); + }); + + it('serves the aggregate it holds when a later load fails', async () => { + jest.spyOn(logService, 'getLatestFinancialLog').mockResolvedValue(logEntry()); + const loaded = await service.getLatestBalance(); + expect(loaded).toMatchObject({ timestamp: new Date('2026-07-14T12:00:00Z') }); + + // Nothing ages the entry out here, so force the load the store would do a minute later. + jest.spyOn(logService, 'getLatestFinancialLog').mockRejectedValue(new Error('database unavailable')); + + await expect(service.refreshLatestBalance()).rejects.toThrow('database unavailable'); + await expect(service.getLatestBalance()).resolves.toBe(loaded); }); }); @@ -464,7 +471,7 @@ describe('DashboardFinancialService', () => { await refreshFrom(timestamp, assetLog, balancesByFinancialType, assets); - expect(latestBalanceStore.set).toHaveBeenCalledWith(expected); + await expect(service.getLatestBalance()).resolves.toEqual(expected); }); it('treats a priceless asset (priceChf: null) neutrally: neither its own blockchain group nor a shared one is skewed', async () => { @@ -524,77 +531,30 @@ describe('DashboardFinancialService', () => { await refreshFrom(timestamp, assetLog, balancesByFinancialType, assets); - expect(latestBalanceStore.set).toHaveBeenCalledWith(expected); + await expect(service.getLatestBalance()).resolves.toEqual(expected); }); - it.each([CronRole.ALL, CronRole.API])('fills the store at start-up in the %s role', async (role) => { - // getLatestBalance answers from the store and nothing else, so until the first fill the - // endpoint has no value to return. - Config.cronRole = role; + it('keeps the aggregate it holds when the newest entry cannot be parsed', async () => { + // The job runs every minute. A single malformed entry must not replace a good value with + // nothing, and the endpoint must keep answering while someone looks at the entry. jest.spyOn(logService, 'getLatestFinancialLog').mockResolvedValue(logEntry()); + const loaded = await service.getLatestBalance(); + expect(loaded).toMatchObject({ timestamp: new Date('2026-07-14T12:00:00Z') }); - service.onModuleInit(); - await new Promise(process.nextTick); - - expect(latestBalanceStore.set).toHaveBeenCalled(); - }); - - it('does not fill it in the worker role, where nothing reads the store', async () => { - Config.cronRole = CronRole.WORKER; - const getLatestFinancialLogSpy = jest.spyOn(logService, 'getLatestFinancialLog'); - - service.onModuleInit(); - await new Promise(process.nextTick); - - expect(getLatestFinancialLogSpy).not.toHaveBeenCalled(); - expect(latestBalanceStore.set).not.toHaveBeenCalled(); - }); - - it('stays off at start-up when the process flag is off', async () => { - // The job carries Process.LATEST_BALANCE_CACHE, and this call bypasses the scheduler that - // applies it. Without the check, switching the job off still leaves one run per deployment. - Config.cronRole = CronRole.API; - jest.spyOn(ProcessService, 'DisabledProcess').mockReturnValue(true); - const getLatestFinancialLogSpy = jest.spyOn(logService, 'getLatestFinancialLog'); - - service.onModuleInit(); - await new Promise(process.nextTick); - - expect(getLatestFinancialLogSpy).not.toHaveBeenCalled(); - expect(latestBalanceStore.set).not.toHaveBeenCalled(); - }); - - it('logs the failure of the start-up fill instead of swallowing it', async () => { - // The scheduled run retries a minute later, so this must not throw - but a silent failure - // would leave the endpoint empty with no trace of why. - Config.cronRole = CronRole.API; - jest.spyOn(logService, 'getLatestFinancialLog').mockRejectedValue(new Error('database unavailable')); - const errorSpy = jest.spyOn(service['logger'], 'error').mockImplementation(() => undefined); - - service.onModuleInit(); - await new Promise(process.nextTick); + jest + .spyOn(logService, 'getLatestFinancialLog') + .mockResolvedValue({ id: 2, created: new Date(), message: 'not json' } as Log); - expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining('start-up'), expect.any(Error)); + await expect(service.refreshLatestBalance()).rejects.toThrow('id 2'); + await expect(service.getLatestBalance()).resolves.toBe(loaded); }); - it('leaves the store untouched when there is no log entry yet', async () => { + it('leaves the store empty when there is no log entry yet', async () => { jest.spyOn(logService, 'getLatestFinancialLog').mockResolvedValue(undefined); await service.refreshLatestBalance(); - expect(latestBalanceStore.set).not.toHaveBeenCalled(); - }); - - it('leaves the store untouched on an unparsable log entry instead of throwing', async () => { - // The job runs every minute. A single malformed entry must not take the endpoint down with - // it, nor replace a good value with a broken one. - jest - .spyOn(logService, 'getLatestFinancialLog') - .mockResolvedValue({ id: 1, created: new Date(), message: 'not json' } as Log); - - await expect(service.refreshLatestBalance()).resolves.toBeUndefined(); - - expect(latestBalanceStore.set).not.toHaveBeenCalled(); + await expect(service.getLatestBalance()).resolves.toBeUndefined(); }); it('does not query assets when the log entry holds none', async () => { @@ -606,7 +566,25 @@ describe('DashboardFinancialService', () => { await service.refreshLatestBalance(); expect(getAssetsByIdSpy).not.toHaveBeenCalled(); - expect(latestBalanceStore.set).toHaveBeenCalled(); + await expect(service.getLatestBalance()).resolves.toBeDefined(); + }); + + it('replaces an entry the read would still have served', async () => { + // What the job is for: the request path in this process finds a current value instead of + // loading one itself. + jest.spyOn(logService, 'getLatestFinancialLog').mockResolvedValue(logEntry()); + await service.getLatestBalance(); + + jest.spyOn(logService, 'getLatestFinancialLog').mockResolvedValue({ + id: 2, + created: new Date('2026-07-14T12:01:00Z'), + message: JSON.stringify({ assets: {}, balancesByFinancialType: {} }), + } as Log); + await service.refreshLatestBalance(); + + await expect(service.getLatestBalance()).resolves.toMatchObject({ + timestamp: new Date('2026-07-14T12:01:00Z'), + }); }); }); }); diff --git a/src/subdomains/supporting/dashboard/dashboard-financial.service.ts b/src/subdomains/supporting/dashboard/dashboard-financial.service.ts index 742b97f7eb..b5344a13e5 100644 --- a/src/subdomains/supporting/dashboard/dashboard-financial.service.ts +++ b/src/subdomains/supporting/dashboard/dashboard-financial.service.ts @@ -1,10 +1,8 @@ -import { Injectable, OnModuleInit } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; -import { Config, CronRole } from 'src/config/config'; import { Asset } from 'src/shared/models/asset/asset.entity'; import { AssetService } from 'src/shared/models/asset/asset.service'; -import { DfxLogger } from 'src/shared/services/dfx-logger'; -import { DisabledProcess, Process } from 'src/shared/services/process.service'; +import { Process } from 'src/shared/services/process.service'; import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { RefRewardService } from '../../core/referral/reward/services/ref-reward.service'; import { AssetLog, BalancesByFinancialType, FinanceLog } from '../log/dto/log.dto'; @@ -23,9 +21,7 @@ import { import { LatestBalanceStore } from './latest-balance.store'; @Injectable() -export class DashboardFinancialService implements OnModuleInit { - private readonly logger = new DfxLogger(DashboardFinancialService); - +export class DashboardFinancialService { constructor( private readonly logService: LogService, private readonly assetService: AssetService, @@ -33,28 +29,6 @@ export class DashboardFinancialService implements OnModuleInit { private readonly latestBalanceStore: LatestBalanceStore, ) {} - onModuleInit() { - // Fills the store once at start-up instead of leaving it empty until the first scheduled run: - // getLatestBalance answers from the store and nothing else, so until the first fill the - // endpoint has no value to return. The two conditions below are the ones the scheduler applies - // to the job itself, and this call bypasses the scheduler entirely. - // - // The role first: the job is scoped `api` because a request path is the only reader of the - // store. In the worker the aggregation would be spent on a value no request there can read. - if (Config.cronRole === CronRole.WORKER) return; - - // Then the flag: a job switched off through DISABLED_PROCESSES has to stay off, including at - // start-up. Otherwise switching it off still leaves one run per deployment. - if (DisabledProcess(Process.LATEST_BALANCE_CACHE)) return; - - void this.refreshLatestBalance().catch((e) => - // Not rethrown: a failed first fill leaves the store empty, which the endpoint already - // handles, and the scheduled run retries a minute later. Swallowing it silently would hide - // why the endpoint is empty in the meantime. - this.logger.error('Failed to fill the latest balance store at start-up:', e), - ); - } - async getFinancialLog(from?: Date, dailySample?: boolean, includeByType?: boolean): Promise { // BTC price is projected in SQL and needs btcAssetId as a parameter, so resolve getBtcCoin first. // One extra sequential roundtrip vs the previous Promise.all, judged negligible against the @@ -150,22 +124,24 @@ export class DashboardFinancialService implements OnModuleInit { } } + /** + * Answers from LatestBalanceStore, which loads through `loadLatestBalance` when it has no entry + * or the one it holds has aged out. The load is what makes this correct in a process that never + * runs the refresh below - see the store for why there is such a process. + */ async getLatestBalance(): Promise { - return this.latestBalanceStore.get(); + return this.latestBalanceStore.get(() => this.loadLatestBalance()); } /** - * Fills LatestBalanceStore from the most recent FinancialDataLog entry, so that - * GET /v1/dashboard/financial/latest keeps answering from process memory without touching the - * database - see getLatestBalance below, which reads the store and nothing else. + * Keeps the entry in LatestBalanceStore warm, so that GET /v1/dashboard/financial/latest finds a + * current value in this process instead of loading one itself. * - * Scope Api, because the store it fills is a field of this service and its reader is the - * endpoint above. It only reads: LogJobService writes the entry, this parses it and aggregates - * - once a minute, outside any request. - * - * Not a fallback for an empty store: it runs on every tick regardless of the store's contents, - * so the path is the normal one rather than one reached only after a failure. The cost is one - * read per tick, in every role that registers it. + * Scope Api, because the store it fills is a field of this service and its reader is the endpoint + * above. It only reads: LogJobService writes the entry, this parses it and aggregates - once a + * minute, outside any request. Refresh only, never the sole filler: the job is leased, so in a + * deployment with several API processes it runs in one of them per tick, and the request path is + * what fills the rest. * * Every minute rather than the 15 minutes CONTRIBUTING prefers, because that is the interval * LogJobService already writes the underlying entry at (TRADING_LOG, EVERY_MINUTE). A longer @@ -173,29 +149,33 @@ export class DashboardFinancialService implements OnModuleInit { */ @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.API, process: Process.LATEST_BALANCE_CACHE }) async refreshLatestBalance(): Promise { + await this.latestBalanceStore.refresh(() => this.loadLatestBalance()); + } + + /** + * Builds the response from the most recent FinancialDataLog entry. `undefined` when there is no + * entry at all, which is the honest answer for a database that has never had one. + */ + private async loadLatestBalance(): Promise { const latest = await this.logService.getLatestFinancialLog(); - if (!latest) return; + if (!latest) return undefined; let financeLog: FinanceLog; try { financeLog = JSON.parse(latest.message); } catch (e) { - this.logger.error(`Failed to parse the latest financial log (id ${latest.id}):`, e); - return; + // Raised rather than returned as `undefined`: the store keeps the aggregate it has, so one + // malformed entry does not replace a good value with nothing. The refresh job reports it. + throw new Error(`Failed to parse the latest financial log (id ${latest.id})`, { cause: e }); } const assets = financeLog.assets ? await this.assetService.getAssetsById(Object.keys(financeLog.assets).map(Number)) : []; - this.latestBalanceStore.set( - this.buildLatestBalance(latest.created, financeLog.assets, financeLog.balancesByFinancialType, assets), - ); + return this.buildLatestBalance(latest.created, financeLog.assets, financeLog.balancesByFinancialType, assets); } - // Unchanged aggregation that used to run inline in getLatestBalance against a freshly parsed - // FinancialDataLog message and a database asset lookup: identical logic, moved here verbatim: only - // its inputs changed (passed in directly instead of JSON.parse(latest.message) / assetService.getAssetsById). private buildLatestBalance( timestamp: Date, assetLog: AssetLog, diff --git a/src/subdomains/supporting/dashboard/latest-balance.store.ts b/src/subdomains/supporting/dashboard/latest-balance.store.ts index 964918d1cb..9e1cb3ceb6 100644 --- a/src/subdomains/supporting/dashboard/latest-balance.store.ts +++ b/src/subdomains/supporting/dashboard/latest-balance.store.ts @@ -1,28 +1,39 @@ import { Injectable } from '@nestjs/common'; +import { AsyncCache, CacheItemResetPeriod } from 'src/shared/utils/async-cache'; import { LatestBalanceResponseDto } from './dto/financial-log.dto'; +const LATEST_BALANCE_KEY = 'latest'; + /** - * Holds the single most recent LatestBalanceResponseDto, written once a minute by - * DashboardFinancialService.refreshLatestBalance from the newest FinancialDataLog entry, and read - * by GET /v1/dashboard/financial/latest. Exactly one entry, replaced wholesale on every job run: - * no TTL, no eviction, no size cap. + * Holds the single most recent LatestBalanceResponseDto, derived from the newest FinancialDataLog + * entry and read by GET /v1/dashboard/financial/latest. Exactly one entry, replaced wholesale: no + * eviction, no size cap. * - * The store is process-local, and so is the job filling it: it carries CronScope.API, so it runs - * in whichever process serves the requests reading it. There is one writer per process and no - * cross-process state to reconcile - both derive the same value from the same row. + * The store is process-local, so every process answering that request needs its own copy - and the + * job cannot be what puts it there. DashboardFinancialService.refreshLatestBalance is scoped `api` + * and therefore runs under a lease: with more than one API process, one of them takes the tick and + * the others do not run the job at all. A store that only the job filled would stay at whatever + * the losing processes started with. * - * Empty (undefined) until the first job run after process start; the read side must not fall back - * to the database in that window (see DashboardFinancialService.getLatestBalance). + * So the read fills it: `get` loads through the loader it is given whenever there is no entry or + * the one it holds has aged out - what CONTRIBUTING asks of a cache a request path reads. The job keeps + * the entry warm in the process that took the tick, so requests there never wait for the load. */ @Injectable() export class LatestBalanceStore { - private value: LatestBalanceResponseDto | undefined; + private readonly cache = new AsyncCache(CacheItemResetPeriod.EVERY_1_MINUTE); - get(): LatestBalanceResponseDto | undefined { - return this.value; + /** + * Serves the entry, loading it through `load` when there is none or it has aged out. A failing + * load leaves whatever is there in place and is not raised at the request: an aggregate a minute + * older answers better than an error, and the refresh below is what reports the failure. + */ + async get(load: () => Promise): Promise { + return this.cache.get(LATEST_BALANCE_KEY, load, undefined, true); } - set(value: LatestBalanceResponseDto): void { - this.value = value; + /** Replaces the entry regardless of its age, and raises what `load` throws so the job reports it. */ + async refresh(load: () => Promise): Promise { + await this.cache.get(LATEST_BALANCE_KEY, load, () => true); } } From 70ddb57fd40ebc40736656578564fc7c6fc12db2 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:38 +0200 Subject: [PATCH 64/86] Drop a repository-wide count from a test comment The comment stated how many declarations carry the timeout and called it the longest value in the repository. No test holds either claim, and the next job added makes both wrong without a word. --- src/shared/services/__tests__/dfx-cron.service.spec.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/shared/services/__tests__/dfx-cron.service.spec.ts b/src/shared/services/__tests__/dfx-cron.service.spec.ts index f0ce908762..96ce71f9df 100644 --- a/src/shared/services/__tests__/dfx-cron.service.spec.ts +++ b/src/shared/services/__tests__/dfx-cron.service.spec.ts @@ -228,11 +228,11 @@ describe('DfxCronService', () => { }); it('does not turn a long job timeout into a long lease', async () => { - // The lease used to expire when the job's own timeout did. Nineteen @DfxCron declarations - // carry `timeout: 7200` — seconds, per LockClass, and the longest value in this repository — - // so a process killed mid-run left the row behind for two hours and its successor sat the - // job out for that long, silently. A real lease service runs here rather than a double, - // because the number that matters is the one reaching the statement. + // The lease used to expire when the job's own timeout did. `timeout` is in seconds, per + // LockClass, so the 7200 declared below left the row behind for two hours after a process + // was killed mid-run, and its successor sat the job out for that long, silently. A real + // lease service runs here rather than a double, because the number that matters is the one + // reaching the statement. process.env.CRON_ROLE = 'worker'; new ConfigService(GetConfig()); From 534a6651e19d259ca42c44b0708f9da469359879 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:40 +0200 Subject: [PATCH 65/86] Say only what the completion path guarantees The comment claimed the activations were closed before the transition, which holds only when the quote reached a state PaymentQuoteFinalStates lists - a completion threshold of TX_RECEIVED does not. --- .../payment-link/services/payment-link-payment.service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts index 0df2583175..79e886172d 100644 --- a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts @@ -677,8 +677,8 @@ export class PaymentLinkPaymentService { payment, PaymentLinkPaymentStatus.COMPLETED, // The count belongs to the transition: `doSave` below is the only other thing that - // writes it, and a completed payment no job looks at again would keep whatever count it - // had when the caller stopped. The activations are already closed above. + // writes it, and no job looks at a completed payment again, so a count left behind by a + // caller that stopped between the two would stay wrong. async (manager) => { await manager.update(PaymentLinkPayment, payment.id, { txCount }); }, From 2fa9d5cd6feadd7f6b8176f67c934ecd7f783b6d Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:41 +0200 Subject: [PATCH 66/86] Bound the double run by what actually bounds it The lease decides who runs the job, not how long two runs overlap; in the shutdown case that is the container's stop grace period. --- src/shared/services/__tests__/cron-lease.service.spec.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/shared/services/__tests__/cron-lease.service.spec.ts b/src/shared/services/__tests__/cron-lease.service.spec.ts index 9a8141e874..57159079f7 100644 --- a/src/shared/services/__tests__/cron-lease.service.spec.ts +++ b/src/shared/services/__tests__/cron-lease.service.spec.ts @@ -248,8 +248,9 @@ describe('CronLeaseService', () => { it('does not take the lease away from a job that is still running', async () => { // The dangerous direction. Releasing on SIGTERM would let the successor claim the lease and - // start the same job while this process keeps working on it for the rest of its stop grace - // period — the double run the lease exists to keep short. + // start the same job while this process keeps working on it — a double run for as long as + // the container leaves this process alive, which is the stop grace period and nothing the + // lease has a say in. Holding the lease is what keeps the successor out of it. jest.useFakeTimers({ doNotFake: ['setImmediate', 'nextTick'] }); try { From 7d84532faa606b8062bd4f7de60b9cda3fcf6d14 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:42 +0200 Subject: [PATCH 67/86] Pin the shared load the empty store depends on AsyncCache hands a running update to everyone who asks for the entry while it is in flight, which is what keeps a restart from starting one aggregation per request. The read relies on it and does not implement it, so it is asserted rather than assumed. --- .../dashboard-financial.service.spec.ts | 20 +++++++++++++++++++ .../dashboard/latest-balance.store.ts | 4 ++++ 2 files changed, 24 insertions(+) diff --git a/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts b/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts index e0bbc0c1df..e9f9d74943 100644 --- a/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts +++ b/src/subdomains/supporting/dashboard/__tests__/dashboard-financial.service.spec.ts @@ -370,6 +370,26 @@ describe('DashboardFinancialService', () => { expect(getLatestFinancialLogSpy).toHaveBeenCalledTimes(1); }); + it('runs one load for the requests that arrive together on an empty store', async () => { + // The state a deploy leaves behind: every process starts with an empty store, so the requests + // that arrive first all miss it at once. They join the load already running instead of each + // starting an aggregation of their own — the property this read depends on and does not + // implement itself, so it is asserted here rather than assumed. + let load: (log: Log) => void; + const getLatestFinancialLogSpy = jest + .spyOn(logService, 'getLatestFinancialLog') + .mockReturnValue(new Promise((resolve) => (load = resolve))); + + const first = service.getLatestBalance(); + const second = service.getLatestBalance(); + + load(logEntry()); + + await expect(first).resolves.toMatchObject({ timestamp: new Date('2026-07-14T12:00:00Z') }); + await expect(second).resolves.toMatchObject({ timestamp: new Date('2026-07-14T12:00:00Z') }); + expect(getLatestFinancialLogSpy).toHaveBeenCalledTimes(1); + }); + it('serves the aggregate it holds when a later load fails', async () => { jest.spyOn(logService, 'getLatestFinancialLog').mockResolvedValue(logEntry()); const loaded = await service.getLatestBalance(); diff --git a/src/subdomains/supporting/dashboard/latest-balance.store.ts b/src/subdomains/supporting/dashboard/latest-balance.store.ts index 9e1cb3ceb6..d38c7e3935 100644 --- a/src/subdomains/supporting/dashboard/latest-balance.store.ts +++ b/src/subdomains/supporting/dashboard/latest-balance.store.ts @@ -27,6 +27,10 @@ export class LatestBalanceStore { * Serves the entry, loading it through `load` when there is none or it has aged out. A failing * load leaves whatever is there in place and is not raised at the request: an aggregate a minute * older answers better than an error, and the refresh below is what reports the failure. + * + * Requests that miss together share the one load: `AsyncCache` keeps the running update on the + * entry and hands it to everyone who asks while it is in flight. That is what a restart depends + * on — the store starts empty in every process, so the requests arriving first all miss at once. */ async get(load: () => Promise): Promise { return this.cache.get(LATEST_BALANCE_KEY, load, undefined, true); From 0a865240d20556c9d06ffc116b5f60a52e73ed42 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:43 +0200 Subject: [PATCH 68/86] Read a span before now, not a mark that outruns uncommitted rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The device delivery kept a per-device high-water mark and moved it to the newest `updated` it had read. A payment is stamped by the statement that writes it and becomes readable only when that write commits, so a mark past the stamp of a row still in flight asks for `updated > since` and never sees that row again. Two rows stamped alike are the plainest case; any write outlived by the read beside it does the same. A span measured against the present cannot skip a row that way, and it bounds the read for a connection of any age — the mark only did so while payments kept arriving. What the span admits twice, a record of the states already sent answers for, per payment rather than one slot per device, and that record is dropped with the payments that leave the window. --- .../payment-link-payment.service.spec.ts | 126 +++++++++++++++--- .../services/payment-link-payment.service.ts | 97 ++++++++++---- 2 files changed, 173 insertions(+), 50 deletions(-) diff --git a/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts b/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts index f580408733..48c3dd7f21 100644 --- a/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts +++ b/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts @@ -1,6 +1,7 @@ import { ConfigService, GetConfig } from 'src/config/config'; import { BlockchainRegistryService } from 'src/integration/blockchain/shared/services/blockchain-registry.service'; import { FiatService } from 'src/shared/models/fiat/fiat.service'; +import { Util } from 'src/shared/utils/util'; import { EntityManager, In } from 'typeorm'; import { PaymentDevice, PaymentLinkPayment } from '../../entities/payment-link-payment.entity'; import { PaymentQuote } from '../../entities/payment-quote.entity'; @@ -23,13 +24,17 @@ describe('PaymentLinkPaymentService', () => { let paymentQuoteService: jest.Mocked; let paymentActivationService: jest.Mocked; + /** + * Times are relative to now because the delivery read is: it asks for a span before the present, + * so a payment stamped at a fixed calendar date would sit outside every window these tests set up. + */ function payment(values: Partial): PaymentLinkPayment { return Object.assign(new PaymentLinkPayment(), { id: 7, status: PaymentLinkPaymentStatus.PENDING, mode: PaymentLinkPaymentMode.SINGLE, txCount: 0, - updated: new Date('2026-01-01T10:00:00Z'), + updated: Util.secondsBefore(1), link: {}, ...values, }); @@ -57,10 +62,26 @@ describe('PaymentLinkPaymentService', () => { */ let sockets: Map; - function connect(deviceId: string, since = new Date('2026-01-01T09:00:00Z')): void { + function connect(deviceId: string, since = Util.minutesBefore(5)): void { sockets.set(deviceId, since); } + /** The rows the database holds for the delivery read below. */ + let rows: PaymentLinkPayment[]; + + /** + * Answers the delivery read out of `rows`, honouring the window each clause carries. Which rows a + * window admits is the whole subject of the tests that use this, so a mock that returned a fixed + * list whatever it was asked for would prove nothing about them. + */ + function findInWindow(options: unknown): PaymentLinkPayment[] { + const windows = (options as { where: { deviceId: string; updated: { value: Date } }[] }).where; + + return rows.filter((row) => + windows.some((window) => row.deviceId === window.deviceId && row.updated > window.updated.value), + ); + } + /** * The row the transition competes for, and the only thing that says whether a caller won: the * manager below applies an update exactly when its criteria still match, as the database does. @@ -106,6 +127,7 @@ describe('PaymentLinkPaymentService', () => { beforeEach(() => { row = payment({ id: 7 }); + rows = []; managerUpdates = []; paymentLinkPaymentRepo = { @@ -193,7 +215,6 @@ describe('PaymentLinkPaymentService', () => { status: PaymentLinkPaymentStatus.COMPLETED, deviceId: 'pos-1', deviceCommand: 'show-paid', - updated: new Date('2026-01-01T11:00:00Z'), }), ]); await service.deliverPaymentUpdates(); @@ -202,6 +223,8 @@ describe('PaymentLinkPaymentService', () => { }); it('should send the same payment state to a device once', async () => { + // The window admits the same payment on every tick it spans, so what keeps the command from + // being repeated is the record of what was sent, not the read. const seen = devices(); connect('pos-1'); @@ -211,7 +234,6 @@ describe('PaymentLinkPaymentService', () => { status: PaymentLinkPaymentStatus.COMPLETED, deviceId: 'pos-1', deviceCommand: 'show-paid', - updated: new Date('2026-01-01T11:00:00Z'), }), ]); await service.deliverPaymentUpdates(); @@ -220,6 +242,23 @@ describe('PaymentLinkPaymentService', () => { expect(seen).toHaveLength(1); }); + it('should send both payments of a device that carry the same state', async () => { + // One slot per device could hold only the later of the two, and the next tick would then find + // the earlier one undelivered again — the two would take turns evicting each other for as + // long as the window spans both. + const seen = devices(); + connect('pos-1'); + + const paid = (id: number) => + payment({ id, status: PaymentLinkPaymentStatus.COMPLETED, deviceId: 'pos-1', deviceCommand: 'show-paid' }); + + paymentLinkPaymentRepo.find.mockResolvedValue([paid(7), paid(8)]); + await service.deliverPaymentUpdates(); + await service.deliverPaymentUpdates(); + + expect(seen).toHaveLength(2); + }); + it('should stop looking for a device the moment the gateway no longer holds it', async () => { // Nothing tells the service the device went away, and nothing has to: it reads the connected // devices on every delivery, so a device that is gone simply stops appearing. @@ -235,35 +274,80 @@ describe('PaymentLinkPaymentService', () => { }); it('should ask each device for its own window, not for the oldest one of all', async () => { - // A single minimum across all devices lets the quietest one set the window for everyone: the - // busy device is then re-read from the point the quiet one connected, on every tick. + // A single minimum across all devices lets the oldest connection set the window for everyone: + // a device that connected moments ago is then read from an hour before it existed, on every + // tick. The window is also what bounds the read, so it has to hold for the long connection + // too — it may not reach back to the day that one was accepted. + const justConnected = new Date(); + connect('pos-1', Util.minutesBefore(90)); + connect('pos-2', justConnected); + + await service.deliverPaymentUpdates(); + + const { where } = paymentLinkPaymentRepo.find.mock.calls[0][0]; + const windows = new Map((where as { deviceId: string; updated: { value: Date } }[]).map((w) => [w.deviceId, w])); + + // The long connection is read from the span, not from the day it was accepted; the newcomer + // from its own connection time, because nothing before it is this process's to deliver. One + // shared window could not express both. + expect(windows.get('pos-1').updated.value.getTime()).toBeGreaterThan(Util.minutesBefore(2).getTime()); + expect(windows.get('pos-2').updated.value).toEqual(justConnected); + }); + + it('should still deliver a payment that becomes visible after one stamped alike', async () => { + // A payment carries the stamp its write gave it and appears only once that write commits, so + // a read running beside it sees the stamp of a row it cannot see yet. A mark advanced to the + // newest stamp read would ask for something strictly newer on the next tick and never see + // that row at all. const seen = devices(); - connect('pos-1', new Date('2026-01-01T09:00:00Z')); + connect('pos-1'); + paymentLinkPaymentRepo.find.mockImplementation(async (options) => findInWindow(options)); - paymentLinkPaymentRepo.find.mockResolvedValue([ + const updated = Util.secondsBefore(1); + const paid = (id: number) => payment({ - id: 7, + id, status: PaymentLinkPaymentStatus.COMPLETED, deviceId: 'pos-1', deviceCommand: 'show-paid', - updated: new Date('2026-01-01T11:00:00Z'), - }), - ]); + updated, + }); + + rows = [paid(7)]; await service.deliverPaymentUpdates(); expect(seen).toHaveLength(1); - // A second device joins, connected long before anything happened on the first one. - connect('pos-2', new Date('2026-01-01T08:00:00Z')); - paymentLinkPaymentRepo.find.mockClear(); + // The slower write commits. Its row was stamped at the same moment as the one already read. + rows = [paid(7), paid(8)]; await service.deliverPaymentUpdates(); - const { where } = paymentLinkPaymentRepo.find.mock.calls[0][0]; - const windows = new Map((where as { deviceId: string; updated: { value: Date } }[]).map((w) => [w.deviceId, w])); + // Delivered — and the payment of the first tick was not sent a second time. + expect(seen).toHaveLength(2); + }); + + it('should forget a payment the window has left behind', async () => { + // What bounds the record of delivered payments is the window: a payment past its far end + // cannot come back from the read, so nothing is kept for it. Without that, a device connected + // all day would accumulate an entry per payment it ever saw. + connect('pos-1'); + paymentLinkPaymentRepo.find.mockImplementation(async (options) => findInWindow(options)); + + // Delivered directly by the writing process, which does not consult the window. + rows = []; + service['deliverToDevice']( + payment({ + id: 7, + status: PaymentLinkPaymentStatus.COMPLETED, + deviceId: 'pos-1', + deviceCommand: 'show-paid', + updated: Util.minutesBefore(2), + }), + ); + expect(service['deviceDeliveries'].get('pos-1').size).toEqual(1); + + await service.deliverPaymentUpdates(); - // The device that was already delivered to has moved on; the newcomer starts at its own - // connection time. One shared window could not express both. - expect(windows.get('pos-1').updated.value).toEqual(new Date('2026-01-01T11:00:00Z')); - expect(windows.get('pos-2').updated.value).toEqual(new Date('2026-01-01T08:00:00Z')); + expect(service['deviceDeliveries'].get('pos-1').size).toEqual(0); }); }); diff --git a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts index 79e886172d..204c8517c5 100644 --- a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts @@ -53,20 +53,34 @@ export interface ConnectedDevice { since: Date; } -/** How far this process has got in delivering to one device. Never a record of who is connected. */ -interface DeliveryCursor { - /** Payments updated at or before this have been delivered, or predate the oldest connection. */ - since: Date; - /** `:` of the last command delivered, so the same state is sent once. */ - delivered?: string; -} +/** + * How far back the delivery read below looks, per device. + * + * What it replaces is a mark that advanced to the newest `updated` it had read. A row carries the + * stamp the writing statement gave it and becomes readable only when that write commits, so a mark + * moved past the stamp of a row that had not committed yet asks for `updated > since` and never + * sees that row again — the notification is not late, it is gone. Two rows stamped alike are the + * plainest case of it, but any write outlived by the read that ran beside it does the same. + * + * A span measured against the present cannot skip a row that way: it only has to outlast the gap + * between a payment's stamp and its commit, which for the transitions in this service is one + * statement plus the effects that travel with it. It is also what bounds the read — see + * `windowStart`. + */ +const DEVICE_DELIVERY_WINDOW_SECONDS = 60; + +/** + * What this process has delivered to one device: per payment, the wait state last sent for it and + * the `updated` that state was read at. Never a record of who is connected — see `connectedDevices`. + */ +type DeviceDeliveries = Map; @Injectable() export class PaymentLinkPaymentService { private readonly paymentWaitMap = new AsyncMap(this.constructor.name); private readonly waitStates = new Map(); private readonly deviceActivationSubject = new Subject(); - private readonly deviceCursors = new Map(); + private readonly deviceDeliveries = new Map(); /** * Where the connected devices are READ from — set once by PaymentLinkGateway, which owns the @@ -309,6 +323,11 @@ export class PaymentLinkPaymentService { * * Both halves are bounded by what this process actually holds: with no caller waiting and no * device connected they touch the database not at all. + * + * That it reads on a tick rather than subscribing is the choice this makes against CONTRIBUTING's + * "initial fetch + subscription for real-time data": the only subscription available here is the + * RxJS subject above, and that reaches no process but this one. A subscription that cannot see + * the writes it is meant to relay is not one. */ async deliverPaymentUpdates(): Promise { await this.deliverToWaitingCallers(); @@ -332,22 +351,22 @@ export class PaymentLinkPaymentService { private async deliverToConnectedDevices(): Promise { const devices = this.connectedDevices(); - // A cursor is a delivery detail of this process, so it follows the connections rather than + // A delivery record is a detail of this process, so it follows the connections rather than // outliving them. Pruning here rather than on a disconnect notification is the point: nothing // has to be told that a device went away, it simply stops appearing. - for (const deviceId of this.deviceCursors.keys()) { - if (!devices.some((device) => device.id === deviceId)) this.deviceCursors.delete(deviceId); + for (const deviceId of this.deviceDeliveries.keys()) { + if (!devices.some((device) => device.id === deviceId)) this.deviceDeliveries.delete(deviceId); } if (!devices.length) return; - // One window PER DEVICE. Taken as a single minimum across all of them, the device connected - // longest — or simply the quietest — sets the window for every other one, and every tick then - // re-reads what those have already been sent. The condition inside each window is the one the - // direct delivery in doSave runs under, expressed over stored columns: a payment out of - // `Pending`, or a `MULTIPLE`-mode payment that has counted a completed quote. + // One window PER DEVICE. Taken as a single minimum across all of them, a device that connected + // moments ago would be read from the point the oldest connection was accepted. The condition + // inside each window is the one the direct delivery in doSave runs under, expressed over stored + // columns: a payment out of `Pending`, or a `MULTIPLE`-mode payment that has counted a + // completed quote. const where = devices.flatMap((device) => { - const { since } = this.cursorFor(device); + const since = this.windowStart(device); return [ { deviceId: device.id, updated: MoreThan(since), status: Not(PaymentLinkPaymentStatus.PENDING) }, @@ -360,12 +379,34 @@ export class PaymentLinkPaymentService { for (const payment of payments) this.deliverToDevice(payment); } - /** The cursor for a connected device, starting at the age of its oldest open connection. */ - private cursorFor(device: ConnectedDevice): DeliveryCursor { - const cursor = this.deviceCursors.get(device.id) ?? { since: device.since }; - this.deviceCursors.set(device.id, cursor); + /** + * Where the read for one device starts: a fixed span before now, and never before its oldest open + * connection was accepted — nothing older than that connection is this process's to deliver. + * + * The span is the same on every tick, so the read stays the same size however long the connection + * lives, whether or not anything happens on it. What the window admits twice, the record below + * answers for; what it lets past its far end can no longer come back from the read, so the record + * of it goes too. That is what bounds the record: not a cap on its size, but the same window that + * bounds the query. + */ + private windowStart(device: ConnectedDevice): Date { + const windowStart = Util.secondsBefore(DEVICE_DELIVERY_WINDOW_SECONDS); + const since = device.since > windowStart ? device.since : windowStart; + + const delivered = this.deliveriesFor(device.id); + for (const [paymentId, entry] of delivered) { + if (!(entry.updated > since)) delivered.delete(paymentId); + } - return cursor; + return since; + } + + /** What has been delivered to a device so far, empty for one nothing has been sent to yet. */ + private deliveriesFor(deviceId: string): DeviceDeliveries { + const delivered = this.deviceDeliveries.get(deviceId) ?? new Map(); + this.deviceDeliveries.set(deviceId, delivered); + + return delivered; } private resolveWaiters(payment: PaymentLinkPayment): void { @@ -384,14 +425,12 @@ export class PaymentLinkPaymentService { const connected = this.connectedDevices().find((d) => d.id === device.id); if (!connected) return; - const cursor = this.cursorFor(connected); - - const state = `${payment.id}:${payment.waitState}`; - if (state === cursor.delivered) return; + // Per payment rather than one slot per device: the window above holds several payments of the + // same device at once, and a single slot would let two of them take turns evicting each other. + const delivered = this.deliveriesFor(connected.id); + if (delivered.get(payment.id)?.state === payment.waitState) return; - cursor.delivered = state; - // Keeps the window of the query above from growing over the lifetime of a connection. - if (payment.updated > cursor.since) cursor.since = payment.updated; + delivered.set(payment.id, { state: payment.waitState, updated: payment.updated }); this.deviceActivationSubject.next(device); } From fd86d07e5da01d650687471f306bb9e664b49cef Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:45 +0200 Subject: [PATCH 69/86] Assert the transitions run their effects on the manager they were given MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The transaction tests counted the effects but never checked what they ran on, so removing the manager from the calls left them green — and the manager is the whole property: an effect on any other one is a statement that commits whether the transition does or not. --- .../payment-link-payment.service.spec.ts | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts b/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts index 48c3dd7f21..3c335e2a9b 100644 --- a/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts +++ b/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts @@ -113,9 +113,19 @@ describe('PaymentLinkPaymentService', () => { managerUpdates.push(update); - return { update } as unknown as EntityManager; + const manager = { update } as unknown as EntityManager; + managers.push(manager); + + return manager; } + /** + * The managers handed to the transitions, in order. An effect reached through anything else is + * a statement of its own: it commits whether the transition does or not, which is the half-state + * the transaction exists to rule out — and nothing about the effect itself shows which it was. + */ + let managers: EntityManager[]; + /** Every statement any transition ran, in order, as [criteria, values]. */ function transitions(): [Partial, Partial][] { return managerUpdates.flatMap((update) => @@ -129,6 +139,7 @@ describe('PaymentLinkPaymentService', () => { row = payment({ id: 7 }); rows = []; managerUpdates = []; + managers = []; paymentLinkPaymentRepo = { find: jest.fn().mockResolvedValue([]), @@ -452,6 +463,17 @@ describe('PaymentLinkPaymentService', () => { ]); }); + it('should run the effects of an expiry on the manager of its transition', async () => { + // The effects are what the transition carries with it. Reached through the repository's own + // manager instead, they would be statements outside it: committed while the status update + // rolls back, or committed after it and lost when the caller stops in between. + await service.expirePayment(payment({ id: 7 })); + + expect(managers).toHaveLength(1); + expect(paymentQuoteService.cancelAllForPayment).toHaveBeenCalledWith(7, managers[0]); + expect(paymentActivationService.closeAllForPayment).toHaveBeenCalledWith(7, managers[0]); + }); + it('should not expire a second time when another process took the transition', async () => { row.status = PaymentLinkPaymentStatus.EXPIRED; @@ -469,7 +491,9 @@ describe('PaymentLinkPaymentService', () => { expect(transitions()).toEqual([ [{ id: 7, status: PaymentLinkPaymentStatus.PENDING }, { status: PaymentLinkPaymentStatus.CANCELLED }], ]); - expect(paymentQuoteService.cancelAllForPayment).toHaveBeenCalledTimes(1); + expect(managers).toHaveLength(1); + expect(paymentQuoteService.cancelAllForPayment).toHaveBeenCalledWith(7, managers[0]); + expect(paymentActivationService.closeAllForPayment).toHaveBeenCalledWith(7, managers[0]); }); it('should not cancel a payment the worker expired in between', async () => { @@ -566,6 +590,10 @@ describe('PaymentLinkPaymentService', () => { [{ id: 7, status: PaymentLinkPaymentStatus.PENDING }, { status: PaymentLinkPaymentStatus.COMPLETED }], [7, { txCount: 1 }], ]); + // Both statements on the one manager the transition was given: a count written through a + // second transaction would commit on its own, which is what carrying it here rules out. + expect(managers).toHaveLength(1); + expect(managers[0].update).toHaveBeenCalledTimes(2); expect(paymentLinkPaymentRepo.save).toHaveBeenCalledTimes(1); }); From deeea817e33407cc4555128754f8bdee62d3c2e0 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:46 +0200 Subject: [PATCH 70/86] Select the delivery on a column no later write can move MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The read that tells a connected device its payment is through was bounded by `updated`, a column every write stamps. Two rounds of review found the same failure through it twice: a mark advanced past a row that had not committed, and then a span against the present that a transaction outliving it walks a row straight past. Both end the same way — the device is never told, and nothing can bring the row back into the read. `expiryDate` is given at insert and never moved afterwards, so a late commit can only make a row appear later, never make it skip. The read now selects on it, one cutoff for every connected device, reaching back past the payment's own end AND past the configured delay before an expiry is acted on — read from the configuration rather than assumed, so raising it cannot silently drop the expiry transition out of the read. The connection time goes with it. What a device is owed follows from the payments, not from when it happened to connect, so a device that reconnects is owed what it was owed before: the delivery record now ages out on the same cutoff the query uses instead of being dropped with the connection, and a device whose entries have all gone leaves with them. --- ...0000-AddPaymentLinkPaymentDeviceIdIndex.js | 4 +- .../__tests__/payment-link.gateway.spec.ts | 27 +-- .../controllers/payment-link.gateway.ts | 15 +- .../payment-link-payment.service.spec.ts | 176 ++++++++++++++---- .../services/payment-link-payment.service.ts | 113 +++++------ 5 files changed, 215 insertions(+), 120 deletions(-) diff --git a/migration/1785620000000-AddPaymentLinkPaymentDeviceIdIndex.js b/migration/1785620000000-AddPaymentLinkPaymentDeviceIdIndex.js index a5d4b65dc6..472fa96393 100644 --- a/migration/1785620000000-AddPaymentLinkPaymentDeviceIdIndex.js +++ b/migration/1785620000000-AddPaymentLinkPaymentDeviceIdIndex.js @@ -13,8 +13,8 @@ * ticks for as long as a device stays connected. * * A single-column index is enough for the shape of the query. It filters `deviceId IN (…)` on the - * handful of devices connected to this process and narrows the result further by `updated` and by - * status — but a device has few payments, so the index gets the row count down to that handful + * handful of devices connected to this process and narrows the result further by `expiryDate` and + * by status — but a device has few payments, so the index gets the row count down to that handful * before the remaining conditions are applied. * * The name is the deterministic one TypeORM's `DefaultNamingStrategy` derives, since CONTRIBUTING diff --git a/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts b/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts index 070535f26b..b257b48fef 100644 --- a/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts +++ b/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts @@ -121,26 +121,17 @@ describe('PaymentLinkGateway', () => { expect(deviceIds()).toEqual([]); }); - it('dates a device by its oldest open connection', () => { - // That date is where the delivery starts reading for a device it has not sent anything to - // yet, so it has to cover every connection, not just the newest. - jest.useFakeTimers({ doNotFake: ['setImmediate', 'nextTick'] }); - - try { - jest.setSystemTime(new Date('2026-01-01T10:00:00Z')); - const first = connect('pos-1'); - - jest.setSystemTime(new Date('2026-01-01T11:00:00Z')); - connect('pos-1'); - - expect(gateway.connectedDevices()[0].since).toEqual(new Date('2026-01-01T10:00:00Z')); + it('names a device once however many connections it holds, and until the last one goes', () => { + // The delivery asks for identities, not for dates: what a device is owed follows from the + // payments themselves. So a second connection adds nothing to say, and closing one of two + // takes nothing away — the device is still reachable through the other. + const first = connect('pos-1'); + connect('pos-1'); - first.fire('close'); + expect(gateway.connectedDevices()).toEqual([{ id: 'pos-1' }]); - expect(gateway.connectedDevices()[0].since).toEqual(new Date('2026-01-01T11:00:00Z')); - } finally { - jest.useRealTimers(); - } + first.fire('close'); + expect(gateway.connectedDevices()).toEqual([{ id: 'pos-1' }]); }); }); diff --git a/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts b/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts index da10e21aaa..19082433f4 100644 --- a/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts +++ b/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts @@ -23,8 +23,6 @@ interface PaymentSocket { /** One open websocket, and what is known about it here. */ interface Connection { socket: PaymentSocket; - /** When this connection was accepted. */ - since: Date; /** Cleared before each ping and set again by the pong; one missed round means it is gone. */ responsive: boolean; } @@ -53,15 +51,12 @@ export class PaymentLinkGateway implements OnGatewayConnection, OnModuleInit { /** * The devices this process can deliver to right now, derived from the sockets it holds open. * - * A device appears here for exactly as long as at least one of its connections is in the map, - * and its `since` is the age of the oldest of those — the point from which a newly connected - * device has not been sent anything yet. + * A device appears here for exactly as long as at least one of its connections is in the map. + * When it connected is deliberately not part of it: the delivery selects payments by their own + * lifetime, so a device that reconnects is owed the same thing it was owed before. */ connectedDevices(): ConnectedDevice[] { - return [...this.clients].map(([id, connections]) => ({ - id, - since: Util.minObj([...connections.values()], 'since').since, - })); + return [...this.clients.keys()].map((id) => ({ id })); } /** @@ -102,7 +97,7 @@ export class PaymentLinkGateway implements OnGatewayConnection, OnModuleInit { const clientId = Util.createUniqueId('client'); const connections = this.clients.get(device) ?? new Map(); - connections.set(clientId, { socket: client, since: new Date(), responsive: true }); + connections.set(clientId, { socket: client, responsive: true }); this.clients.set(device, connections); // Bound to the socket, and to every way it can end: an aborted connection reports `error`, and diff --git a/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts b/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts index 3c335e2a9b..b6f51cd3f3 100644 --- a/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts +++ b/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts @@ -25,8 +25,12 @@ describe('PaymentLinkPaymentService', () => { let paymentActivationService: jest.Mocked; /** - * Times are relative to now because the delivery read is: it asks for a span before the present, - * so a payment stamped at a fixed calendar date would sit outside every window these tests set up. + * Times are relative to now because the delivery read is: it asks for payments whose own end has + * not passed by more than the grace, so a payment dated at a fixed calendar point would sit + * outside every read these tests set up. + * + * `expiryDate` is what the read selects on — deliberately a column no later write moves. The + * default puts a payment inside the read; a test that wants one outside says so. */ function payment(values: Partial): PaymentLinkPayment { return Object.assign(new PaymentLinkPayment(), { @@ -35,6 +39,7 @@ describe('PaymentLinkPaymentService', () => { mode: PaymentLinkPaymentMode.SINGLE, txCount: 0, updated: Util.secondsBefore(1), + expiryDate: Util.minutesAfter(5), link: {}, ...values, }); @@ -60,25 +65,41 @@ describe('PaymentLinkPaymentService', () => { * every delivery, so a device is connected here for exactly as long as this map says so — there * is no register in the service to register with. */ - let sockets: Map; + let sockets: Set; - function connect(deviceId: string, since = Util.minutesBefore(5)): void { - sockets.set(deviceId, since); + function connect(deviceId: string): void { + sockets.add(deviceId); } /** The rows the database holds for the delivery read below. */ let rows: PaymentLinkPayment[]; /** - * Answers the delivery read out of `rows`, honouring the window each clause carries. Which rows a - * window admits is the whole subject of the tests that use this, so a mock that returned a fixed - * list whatever it was asked for would prove nothing about them. + * Answers the delivery read out of `rows`, honouring EVERY condition each clause carries — the + * devices asked for, the cutoff, and the state that makes a payment worth delivering. Which rows + * the read admits is the whole subject of the tests that use this, so a mock that returned a + * fixed list whatever it was asked for would prove nothing about them. */ - function findInWindow(options: unknown): PaymentLinkPayment[] { - const windows = (options as { where: { deviceId: string; updated: { value: Date } }[] }).where; + function findByCutoff(options: unknown): PaymentLinkPayment[] { + const clauses = ( + options as { + where: { + deviceId: { value: string[] }; + expiryDate: { value: Date }; + status?: { value: PaymentLinkPaymentStatus }; + txCount?: { value: number }; + }[]; + } + ).where; return rows.filter((row) => - windows.some((window) => row.deviceId === window.deviceId && row.updated > window.updated.value), + clauses.some( + (clause) => + clause.deviceId.value.includes(row.deviceId) && + row.expiryDate > clause.expiryDate.value && + (clause.status == null || row.status !== clause.status.value) && + (clause.txCount == null || row.txCount > clause.txCount.value), + ), ); } @@ -136,6 +157,10 @@ describe('PaymentLinkPaymentService', () => { let managerUpdates: jest.Mock[]; beforeEach(() => { + // The delivery reads `Config.payment.timeoutDelay` on every tick, deliberately: the span it + // reaches back over follows the configured delay rather than a copy taken at construction. + new ConfigService(GetConfig()); + row = payment({ id: 7 }); rows = []; managerUpdates = []; @@ -162,8 +187,13 @@ describe('PaymentLinkPaymentService', () => { {} as unknown as jest.Mocked, ); - sockets = new Map(); - service.useDeviceSource(() => [...sockets].map(([id, since]) => ({ id, since }))); + sockets = new Set(); + service.useDeviceSource(() => [...sockets].map((id) => ({ id }))); + }); + + afterEach(() => { + // Set by the test that raises it; left behind it would silently widen every later read. + delete process.env.PAYMENT_TIMEOUT_DELAY; }); // --- deliverPaymentUpdates() Tests --- // @@ -284,25 +314,97 @@ describe('PaymentLinkPaymentService', () => { expect(paymentLinkPaymentRepo.find).toHaveBeenCalledTimes(1); }); - it('should ask each device for its own window, not for the oldest one of all', async () => { - // A single minimum across all devices lets the oldest connection set the window for everyone: - // a device that connected moments ago is then read from an hour before it existed, on every - // tick. The window is also what bounds the read, so it has to hold for the long connection - // too — it may not reach back to the day that one was accepted. - const justConnected = new Date(); - connect('pos-1', Util.minutesBefore(90)); - connect('pos-2', justConnected); + it('should ask for every connected device in one read, on one cutoff', async () => { + // The cutoff is a property of the payments, not of the connections, so there is nothing left + // for a per-device clause to express — and one clause per device is what made the read grow + // with the number of connections. + connect('pos-1'); + connect('pos-2'); await service.deliverPaymentUpdates(); const { where } = paymentLinkPaymentRepo.find.mock.calls[0][0]; - const windows = new Map((where as { deviceId: string; updated: { value: Date } }[]).map((w) => [w.deviceId, w])); + const clauses = where as { deviceId: { value: string[] }; expiryDate: { value: Date } }[]; + + expect(clauses).toHaveLength(2); + for (const clause of clauses) { + expect(clause.deviceId.value).toEqual(['pos-1', 'pos-2']); + expect(clause.expiryDate.value).toEqual(clauses[0].expiryDate.value); + } + }); + + it('should reach back past the delay before an expiry is even acted on', async () => { + // processExpiredPayments expires a payment at its expiryDate PLUS this delay, so a cutoff + // measured from the expiryDate alone would drop the payment out of the read before the + // transition it is waiting for has happened. Reading the configured value rather than + // assuming it is what keeps that true when the value changes. + process.env.PAYMENT_TIMEOUT_DELAY = '3600'; + new ConfigService(GetConfig()); + connect('pos-1'); + + await service.deliverPaymentUpdates(); + + const { where } = paymentLinkPaymentRepo.find.mock.calls[0][0]; + const [clause] = where as { expiryDate: { value: Date } }[]; + + expect(clause.expiryDate.value.getTime()).toBeLessThan(Util.minutesBefore(60).getTime()); + }); + + it('should still deliver a payment whose write was outlived by the read beside it', async () => { + // The failure this replaces: a read bounded by `updated` asks for a span before the present, + // and a transaction that stays open longer than that span commits a row whose stamp is + // already past the far end — it is never read again. Selecting on a column no later write + // moves cannot do that: a late commit makes the row appear later, never skip. + const seen = devices(); + connect('pos-1'); + paymentLinkPaymentRepo.find.mockImplementation(async (options) => findByCutoff(options)); + + rows = [ + payment({ + id: 7, + status: PaymentLinkPaymentStatus.COMPLETED, + deviceId: 'pos-1', + deviceCommand: 'show-paid', + // Stamped by a transaction that then took an hour to commit. + updated: Util.minutesBefore(60), + expiryDate: Util.minutesAfter(5), + }), + ]; + await service.deliverPaymentUpdates(); + + expect(seen).toEqual([{ id: 'pos-1', command: 'show-paid' }]); + }); + + it('should owe a reconnecting device exactly what it was owed before', async () => { + // A record tied to the connection is lost with it, and the read then starts at the new + // connection time: what completed just before the reconnect falls between the two and is + // never delivered, while everything before it is delivered again. + const seen = devices(); + connect('pos-1'); + paymentLinkPaymentRepo.find.mockImplementation(async (options) => findByCutoff(options)); - // The long connection is read from the span, not from the day it was accepted; the newcomer - // from its own connection time, because nothing before it is this process's to deliver. One - // shared window could not express both. - expect(windows.get('pos-1').updated.value.getTime()).toBeGreaterThan(Util.minutesBefore(2).getTime()); - expect(windows.get('pos-2').updated.value).toEqual(justConnected); + const paid = (id: number) => + payment({ + id, + status: PaymentLinkPaymentStatus.COMPLETED, + deviceId: 'pos-1', + deviceCommand: 'show-paid', + }); + + rows = [paid(7)]; + await service.deliverPaymentUpdates(); + expect(seen).toHaveLength(1); + + sockets.delete('pos-1'); + // Completes while nothing is connected for it. + rows = [paid(7), paid(8)]; + await service.deliverPaymentUpdates(); + + connect('pos-1'); + await service.deliverPaymentUpdates(); + + // The one it missed, and only that one: the payment of the first tick is not repeated. + expect(seen).toHaveLength(2); }); it('should still deliver a payment that becomes visible after one stamped alike', async () => { @@ -312,7 +414,7 @@ describe('PaymentLinkPaymentService', () => { // that row at all. const seen = devices(); connect('pos-1'); - paymentLinkPaymentRepo.find.mockImplementation(async (options) => findInWindow(options)); + paymentLinkPaymentRepo.find.mockImplementation(async (options) => findByCutoff(options)); const updated = Util.secondsBefore(1); const paid = (id: number) => @@ -336,14 +438,15 @@ describe('PaymentLinkPaymentService', () => { expect(seen).toHaveLength(2); }); - it('should forget a payment the window has left behind', async () => { - // What bounds the record of delivered payments is the window: a payment past its far end - // cannot come back from the read, so nothing is kept for it. Without that, a device connected - // all day would accumulate an entry per payment it ever saw. + it('should forget a payment the read has left behind, and the device with its last one', async () => { + // What bounds the record is the same cutoff the query uses: a payment the read can no longer + // return cannot be delivered again, so nothing is kept for it. Without that, a device + // connected all day would accumulate an entry per payment it ever saw — and a device that + // never comes back would keep a map of its own for good. connect('pos-1'); - paymentLinkPaymentRepo.find.mockImplementation(async (options) => findInWindow(options)); + paymentLinkPaymentRepo.find.mockImplementation(async (options) => findByCutoff(options)); - // Delivered directly by the writing process, which does not consult the window. + // Delivered directly by the writing process, which does not consult the cutoff. rows = []; service['deliverToDevice']( payment({ @@ -351,14 +454,15 @@ describe('PaymentLinkPaymentService', () => { status: PaymentLinkPaymentStatus.COMPLETED, deviceId: 'pos-1', deviceCommand: 'show-paid', - updated: Util.minutesBefore(2), + // Its own end is long past, and past the grace that follows it. + expiryDate: Util.hoursBefore(2), }), ); expect(service['deviceDeliveries'].get('pos-1').size).toEqual(1); await service.deliverPaymentUpdates(); - expect(service['deviceDeliveries'].get('pos-1').size).toEqual(0); + expect(service['deviceDeliveries'].has('pos-1')).toBe(false); }); }); diff --git a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts index 204c8517c5..883c082b78 100644 --- a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts @@ -46,34 +46,43 @@ import { PaymentWebhookService } from './payment-webhook.service'; */ const PAYMENT_WAIT_TIMEOUT_SECONDS = 60; -/** A device this process holds at least one open websocket connection for. */ +/** + * A device this process holds at least one open websocket connection for. + * + * Only the identity: what a device is owed follows from the payments themselves, not from when it + * happened to connect. A device that reconnects is the same device, and the delivery record below + * outlives the connection precisely so that it is treated as one. + */ export interface ConnectedDevice { id: string; - /** When the oldest connection still open for this device was accepted. */ - since: Date; } /** - * How far back the delivery read below looks, per device. + * How long after a payment can no longer expire the delivery read below still asks for it. * - * What it replaces is a mark that advanced to the newest `updated` it had read. A row carries the - * stamp the writing statement gave it and becomes readable only when that write commits, so a mark - * moved past the stamp of a row that had not committed yet asks for `updated > since` and never - * sees that row again — the notification is not late, it is gone. Two rows stamped alike are the - * plainest case of it, but any write outlived by the read that ran beside it does the same. + * The read selects on `expiryDate`, and the reason is which writes can move a column. `updated` is + * stamped by every write; `expiryDate` is given at insert and never mutated afterwards. That is the + * whole difference. Any predicate over a column a later write can move is able to carry a row OUT + * of the read before the read has seen it, and two rounds of review found the same failure twice + * that way: a mark advanced past a row that had not committed, and then a span against the present + * that a transaction outliving it walks a row straight past. A late commit against an immutable + * column can only make a row appear LATER, never make it skip — the row's place in the read was + * fixed at insert, before anything could be late. * - * A span measured against the present cannot skip a row that way: it only has to outlast the gap - * between a payment's stamp and its commit, which for the transitions in this service is one - * statement plus the effects that travel with it. It is also what bounds the read — see - * `windowStart`. + * So this is not a window a payment has to be delivered within. It is how far past a payment's own + * end the read keeps asking, and it is measured from `expiryDate` rather than from now: + * `processExpiredPayments` expires a payment at `expiryDate` plus `Config.payment.timeoutDelay`, so + * the span has to outlast that delay for the expiry transition itself to still be read — which is + * why the cutoff below ADDS the configured delay instead of assuming it away. */ -const DEVICE_DELIVERY_WINDOW_SECONDS = 60; +const DEVICE_DELIVERY_GRACE_SECONDS = 600; /** * What this process has delivered to one device: per payment, the wait state last sent for it and - * the `updated` that state was read at. Never a record of who is connected — see `connectedDevices`. + * the `expiryDate` that decides how long the entry is kept. Never a record of who is connected — + * see `connectedDevices`. */ -type DeviceDeliveries = Map; +type DeviceDeliveries = Map; @Injectable() export class PaymentLinkPaymentService { @@ -350,55 +359,51 @@ export class PaymentLinkPaymentService { private async deliverToConnectedDevices(): Promise { const devices = this.connectedDevices(); + const cutoff = this.deliveryCutoff(); - // A delivery record is a detail of this process, so it follows the connections rather than - // outliving them. Pruning here rather than on a disconnect notification is the point: nothing - // has to be told that a device went away, it simply stops appearing. - for (const deviceId of this.deviceDeliveries.keys()) { - if (!devices.some((device) => device.id === deviceId)) this.deviceDeliveries.delete(deviceId); - } + this.pruneDeliveries(cutoff); if (!devices.length) return; - // One window PER DEVICE. Taken as a single minimum across all of them, a device that connected - // moments ago would be read from the point the oldest connection was accepted. The condition - // inside each window is the one the direct delivery in doSave runs under, expressed over stored - // columns: a payment out of `Pending`, or a `MULTIPLE`-mode payment that has counted a - // completed quote. - const where = devices.flatMap((device) => { - const since = this.windowStart(device); - - return [ - { deviceId: device.id, updated: MoreThan(since), status: Not(PaymentLinkPaymentStatus.PENDING) }, - { deviceId: device.id, updated: MoreThan(since), txCount: MoreThan(0) }, - ]; - }); + // One cutoff for all of them, because it no longer depends on the connection: a payment is read + // until its own end has passed, whoever is listening and since when. The condition is the one + // the direct delivery in doSave runs under, expressed over stored columns: a payment out of + // `Pending`, or a `MULTIPLE`-mode payment that has counted a completed quote. + const deviceIds = devices.map((device) => device.id); + const where = [ + { deviceId: In(deviceIds), expiryDate: MoreThan(cutoff), status: Not(PaymentLinkPaymentStatus.PENDING) }, + { deviceId: In(deviceIds), expiryDate: MoreThan(cutoff), txCount: MoreThan(0) }, + ]; - const payments = await this.paymentLinkPaymentRepo.find({ where, order: { updated: 'ASC' } }); + const payments = await this.paymentLinkPaymentRepo.find({ where, order: { expiryDate: 'ASC' } }); for (const payment of payments) this.deliverToDevice(payment); } /** - * Where the read for one device starts: a fixed span before now, and never before its oldest open - * connection was accepted — nothing older than that connection is this process's to deliver. - * - * The span is the same on every tick, so the read stays the same size however long the connection - * lives, whether or not anything happens on it. What the window admits twice, the record below - * answers for; what it lets past its far end can no longer come back from the read, so the record - * of it goes too. That is what bounds the record: not a cap on its size, but the same window that - * bounds the query. + * The oldest `expiryDate` the read still asks for: a payment's own end, plus the delay before + * `processExpiredPayments` acts on it, plus the grace above. The delay is READ rather than + * assumed — raising `PAYMENT_TIMEOUT_DELAY` past a hard-coded span would otherwise drop the + * expiry transition out of the read without changing a line here. */ - private windowStart(device: ConnectedDevice): Date { - const windowStart = Util.secondsBefore(DEVICE_DELIVERY_WINDOW_SECONDS); - const since = device.since > windowStart ? device.since : windowStart; + private deliveryCutoff(): Date { + return Util.secondsBefore(Config.payment.timeoutDelay + DEVICE_DELIVERY_GRACE_SECONDS); + } - const delivered = this.deliveriesFor(device.id); - for (const [paymentId, entry] of delivered) { - if (!(entry.updated > since)) delivered.delete(paymentId); + /** + * Drops what the read can no longer return. An entry is kept while its payment is still asked + * for, NOT while its device is connected: a device that reconnects finds its record intact and is + * not told a second time about what it already heard. That is also what bounds the record — + * entries age out on the same cutoff the query uses, and a device whose entries have all gone + * leaves with them. + */ + private pruneDeliveries(cutoff: Date): void { + for (const [deviceId, delivered] of this.deviceDeliveries) { + for (const [paymentId, entry] of delivered) { + if (!(entry.expiryDate > cutoff)) delivered.delete(paymentId); + } + if (!delivered.size) this.deviceDeliveries.delete(deviceId); } - - return since; } /** What has been delivered to a device so far, empty for one nothing has been sent to yet. */ @@ -425,12 +430,12 @@ export class PaymentLinkPaymentService { const connected = this.connectedDevices().find((d) => d.id === device.id); if (!connected) return; - // Per payment rather than one slot per device: the window above holds several payments of the + // Per payment rather than one slot per device: the read above holds several payments of the // same device at once, and a single slot would let two of them take turns evicting each other. const delivered = this.deliveriesFor(connected.id); if (delivered.get(payment.id)?.state === payment.waitState) return; - delivered.set(payment.id, { state: payment.waitState, updated: payment.updated }); + delivered.set(payment.id, { state: payment.waitState, expiryDate: payment.expiryDate }); this.deviceActivationSubject.next(device); } From bfe6258584aa1fbe8716a40779c0751a7975be3e Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:48 +0200 Subject: [PATCH 71/86] Say what the lease is claimed for, not how long it is held Four summaries described the lease as held "for the duration of its run" or "for as long as the job runs", and one described the renewal as keeping the claim alive. Each is the short form of a passage that goes on to take it back: the claim can lapse while the job is still working, and losing it stops nothing. A reader who takes only the short form takes away a stronger property than the design has. Shortened rather than qualified, because this is the fourth round in which the same class of sentence has come back. What the lease does is claimed before the start; what happens after that is in "What it does not do", where it already was. --- CONTRIBUTING.md | 4 ++-- migration/1785600000000-AddCronLease.js | 4 ++-- src/shared/services/cron-lease.service.ts | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d3201595e5..405c79d494 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -538,8 +538,8 @@ outside the process infers health from the job still running — a watchdog that looks, once it is off, exactly like the failure it watches for. Without a flag the job runs unconditionally and cannot be switched off without a deploy. -A job scoped `worker` or `api` additionally holds a **lease in the database** for the duration of -its run (`CronLeaseService`). The in-process lock cannot see a second process at all — a missed +A job scoped `worker` or `api` additionally takes a **lease in the database** before it starts +(`CronLeaseService`). The in-process lock cannot see a second process at all — a missed recreate, a second worker from `--scale`, two processes on `all` after a rollback — and the lease is what such a second process has to get past before it may start the job. diff --git a/migration/1785600000000-AddCronLease.js b/migration/1785600000000-AddCronLease.js index 48c65f3fde..9101b02ade 100644 --- a/migration/1785600000000-AddCronLease.js +++ b/migration/1785600000000-AddCronLease.js @@ -14,8 +14,8 @@ * answer. * * This table is what a second process has to get past before it may start such a job: a job scoped - * to exactly one process must hold a row here for the duration of its run, and the row is claimable - * by one process at a time until it expires. The expiry is why this is not an exclusion — if the + * to exactly one process must take a row here before it starts, and the row is claimable by one + * process at a time until it expires. The expiry is why this is not an exclusion — if the * holder can no longer renew, a second process can claim the row while the first is still working, * and how long the two then overlap is not bounded by anything here. See CronLeaseService, "What it * does not do". diff --git a/src/shared/services/cron-lease.service.ts b/src/shared/services/cron-lease.service.ts index 5bdf2b7aef..5adf1e2cd6 100644 --- a/src/shared/services/cron-lease.service.ts +++ b/src/shared/services/cron-lease.service.ts @@ -42,7 +42,7 @@ const RENEWAL_INTERVAL_MS = (LEASE_TTL_SECONDS / 3) * 1000; const SHUTDOWN_GRACE_MS = 10 * 1000; /** - * A lease on a scheduled job, held in the database for as long as the job runs. + * A lease on a scheduled job: the claim a process takes in the database before it starts one. * * `LockClass` keeps its state in a field of a process-local object. That was enough while the API * ran as one process; it cannot see a second one. Since the HTTP process and the worker are split @@ -343,7 +343,7 @@ export class CronLeaseService implements OnModuleInit { } /** - * Keeps the claim for `job` alive while it runs, with one renewal outstanding at a time. + * Renews the claim for `job` while it runs, with one renewal outstanding at a time. * * A fixed interval fires whether or not the previous renewal has come back, and a database that * answers slowly is exactly the situation this has to survive: the attempts pile up, each one From e258492a1bbb983d6c80f11a0c03462fa24cd9e0 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:49 +0200 Subject: [PATCH 72/86] Close the two half-states the transaction boundary still left open MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round seven found two places where a boundary drawn in an earlier round stops one statement short of where the damage is. In `handleQuoteChange`, the activations of a final quote were closed BEFORE the status moved. If the transition then failed, the payment stayed `Pending` with its activations closed for good — and nothing brings that back: the quote is already final, so `checkTxConfirmations` does not return to it, while `processExpiredPayments` only ever asks for `Pending` and would expire a payment that was in fact paid. The close now travels with the transition on the path that has one, and stays where it was on the paths that do not. In `stateRow`, two writers converging an old snapshot row onto `id: 1` could lose the first one's work. Both miss `id: 1`, both queue for the old row, and the second wakes up after the first has created `id: 1` and committed — merging from the old row then writes a state that predates it. Since this path only writes what changed, the lost metric does not come back on its own. The lookup now asks a second time after the wait. Two statements were also promising more than they hold. The delivery grace said a late commit can never make a row skip; that is true of WRITES, and the read still ends somewhere — a transition after that end is missed like any other. The span now reaches an hour instead of ten minutes, chosen against what actually delays the transition: the worker being gone, which the silence alert only reports after seventeen. And the shutdown release is best effort, not a guarantee: a claim still being taken is not in `inFlight`, so an exit in between leaves the row to lapse on its TTL — the bound that always applies. Each of the four is pinned by a test whose counter-proof was seen red. --- .../__tests__/cron-lease.service.spec.ts | 8 +- src/shared/services/cron-lease.service.ts | 8 +- .../__tests__/monitoring.service.spec.ts | 51 +++++++++++ .../core/monitoring/monitoring.service.ts | 20 ++++- .../payment-link-payment.service.spec.ts | 35 ++++++++ .../services/payment-link-payment.service.ts | 84 ++++++++++++------- 6 files changed, 170 insertions(+), 36 deletions(-) diff --git a/src/shared/services/__tests__/cron-lease.service.spec.ts b/src/shared/services/__tests__/cron-lease.service.spec.ts index 57159079f7..5f236811d1 100644 --- a/src/shared/services/__tests__/cron-lease.service.spec.ts +++ b/src/shared/services/__tests__/cron-lease.service.spec.ts @@ -352,8 +352,12 @@ describe('CronLeaseService', () => { it('hands the claim back when shutdown begins while it is being taken', async () => { // Claiming is a round trip, so the guard above can be passed just before shutdown starts. - // The run must not begin under that claim, and must not leave the row behind either: the - // successor would then sit out the full expiry before it could take the job over. + // The run must not begin under that claim, and should not leave the row behind either — the + // successor would then sit out the full expiry for a job that never ran. + // + // Pinned here is the path where the process lives long enough to do it. It is not a promise + // that it always does: the claim is not in `inFlight`, so `shutdown` does not wait for it, + // and an exit in between leaves the row to lapse on its TTL. let answerClaim: (rows: unknown[]) => void; const onQuery = jest.fn().mockImplementation((sql: string) => { if (sql.includes('INSERT INTO')) return new Promise((resolve) => (answerClaim = resolve)); diff --git a/src/shared/services/cron-lease.service.ts b/src/shared/services/cron-lease.service.ts index 5adf1e2cd6..60f8ac68a2 100644 --- a/src/shared/services/cron-lease.service.ts +++ b/src/shared/services/cron-lease.service.ts @@ -249,7 +249,13 @@ export class CronLeaseService implements OnModuleInit { } // Claiming the lease is a round trip, and shutdown can begin during it. Hand the claim straight - // back rather than start under it: the successor can then take the job over immediately. + // back rather than start under it, so the successor does not sit out the expiry for a job that + // never ran. + // + // This is best effort, not a guarantee: a job still inside its claim is not in `inFlight` yet, + // so `shutdown` does not wait for it, and the process can exit before the release below runs. + // What then remains is a claim nobody holds — it lapses within the TTL like any other, which + // is the bound that always applies. The release only ever shortens that wait. if (this.shuttingDown) { await this.release(job).catch(() => undefined); return; diff --git a/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts b/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts index 68245cd6cc..96f58b626a 100644 --- a/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts +++ b/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts @@ -290,5 +290,56 @@ describe('MonitoringService', () => { expect(saved.node.health.data).toEqual({ up: true }); expect(saved.bank.balance.data).toEqual({ chf: 42 }); }); + + it('merges into id 1 when another writer created it while this one waited for the lock', async () => { + // Two writers, one old row and no id 1: both miss id 1, both queue for the old row. The one + // that gets there second wakes up after the first has created id 1 and committed. Merging + // from the OLD row then writes id 1 from a state that predates it — and this path only + // writes what changed, so the first writer's metric does not come back. Asking a second + // time after the wait is what closes that. + const seededById1: SystemState = { + ...persisted, + node: { health: metric({ up: true }, '2020-01-01T00:10:00Z') }, + // What the OTHER writer put there while this one waited. + ledger: { drift: metric({ off: 3 }, '2029-01-01T00:00:00Z') }, + }; + + let idOneExists = false; + const managerFindOne = jest + .fn() + .mockImplementation((_entity: unknown, options: { where?: { id?: number } }) => { + if (options?.where?.id === 1) { + // Missing on the first ask; present once the wait on the old row is over. + return Promise.resolve(idOneExists ? { id: 1, data: JSON.stringify(seededById1) } : null); + } + // The lock on the old row is granted only after the other writer committed. + idOneExists = true; + return Promise.resolve({ id: 7, data: JSON.stringify(persisted) }); + }); + + Object.defineProperty(repo, 'manager', { + value: { + transaction: (run: (m: unknown) => Promise) => + run({ + findOne: managerFindOne, + save: jest.fn().mockImplementation((_e: unknown, row: { id: number; data: string }) => { + written.push(row); + return Promise.resolve(row); + }), + }), + }, + configurable: true, + }); + + await service['mergeIntoStoredState']([['aml', 'freeze']], { + aml: { freeze: metric({ frozen: 0 }, '2030-01-01T00:00:00Z') }, + }); + + const saved = JSON.parse(written[0].data); + + expect(saved.aml.freeze.data).toEqual({ frozen: 0 }); + // The other writer's metric survived — that is the whole point. + expect(saved.ledger.drift.data).toEqual({ off: 3 }); + }); }); }); diff --git a/src/subdomains/core/monitoring/monitoring.service.ts b/src/subdomains/core/monitoring/monitoring.service.ts index 6d59d21f94..e9e25a2ca1 100644 --- a/src/subdomains/core/monitoring/monitoring.service.ts +++ b/src/subdomains/core/monitoring/monitoring.service.ts @@ -266,9 +266,25 @@ export class MonitoringService implements OnModuleInit { if (canonical) return canonical; const fallback = await find({ where: {}, order: { id: 'DESC' }, lock }); - if (fallback) this.logger.warn(`No monitoring state under id 1, using id ${fallback.id} instead`); + if (!fallback) return undefined; + + // Asked a second time, because the answer can have changed while this call waited. With a + // lock, the wait is on the fallback row itself: two writers both miss `id: 1`, both queue for + // the old row, and the one that gets there second wakes up in a world where the first has + // already created `id: 1` and committed. Merging from the old row then writes `id: 1` from a + // state that predates it, and the first writer's metric is gone — not stale, gone, because + // this path only writes what changed and nothing brings the rest back. + // + // Without a lock there is nothing to wait on and this is one extra read on a path that stops + // being taken as soon as the canonical row exists. + if (fallback.id !== 1) { + const converged = await find({ where: { id: 1 }, lock }); + if (converged) return converged; + } + + this.logger.warn(`No monitoring state under id 1, using id ${fallback.id} instead`); - return fallback ?? undefined; + return fallback; } /** diff --git a/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts b/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts index b6f51cd3f3..0f6cc70aae 100644 --- a/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts +++ b/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts @@ -701,6 +701,41 @@ describe('PaymentLinkPaymentService', () => { expect(paymentLinkPaymentRepo.save).toHaveBeenCalledTimes(1); }); + it('should close the activations inside the transition, not before it', async () => { + // The half-state nothing can repair: activations closed while the payment is still + // `Pending`. The quote is already final, so checkTxConfirmations does not come back to + // it, and processExpiredPayments only ever asks for `Pending` — it would expire a payment + // whose activations have been closed for good. Running the close on the transition's own + // manager is what ties the two together. + await service['handleQuoteChange'](completing(), quote); + + expect(paymentActivationService.closeAllForPayment).toHaveBeenCalledWith(7, managers[0]); + }); + + it('should leave the activations open when the transition is lost', async () => { + // Another process took the payment out of `Pending` first. Then this caller performs no + // effect at all — closing activations for a transition it did not win would be the same + // half-state seen from the other side. + row.status = PaymentLinkPaymentStatus.COMPLETED; + + await service['handleQuoteChange'](completing(), quote); + + expect(paymentActivationService.closeAllForPayment).not.toHaveBeenCalled(); + expect(paymentLinkPaymentRepo.save).not.toHaveBeenCalled(); + }); + + it('should still close the activations on the paths that take no transition', async () => { + // A payment that is no longer `Pending` has nothing to transition, but its final quote's + // activations still have to be closed — that path predates the transaction and stays. + const payment = completing(); + payment.status = PaymentLinkPaymentStatus.EXPIRED; + + await service['handleQuoteChange'](payment, quote); + + expect(paymentActivationService.closeAllForPayment).toHaveBeenCalledWith(7, undefined); + expect(transitions()).toEqual([]); + }); + it('should carry the counted quotes into the transition, not only into the save after it', async () => { // A completed payment is looked at by no job, so a count left behind by a caller that // stopped after the transition would stay wrong for good. diff --git a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts index 883c082b78..71f59081e4 100644 --- a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts @@ -65,17 +65,27 @@ export interface ConnectedDevice { * whole difference. Any predicate over a column a later write can move is able to carry a row OUT * of the read before the read has seen it, and two rounds of review found the same failure twice * that way: a mark advanced past a row that had not committed, and then a span against the present - * that a transaction outliving it walks a row straight past. A late commit against an immutable - * column can only make a row appear LATER, never make it skip — the row's place in the read was - * fixed at insert, before anything could be late. + * that a transaction outliving it walks a row straight past. + * + * What the immutable column buys, precisely: no WRITE can move a row out of the read. A late + * commit makes it appear later, never skip, because its place was fixed at insert. What it does + * NOT buy: the read still ends somewhere, so a transition that happens after that end is missed + * like any other. The two are different failures — the first was a property of the predicate and + * is gone; the second is a question of how far the span reaches, and is answered below. * * So this is not a window a payment has to be delivered within. It is how far past a payment's own * end the read keeps asking, and it is measured from `expiryDate` rather than from now: * `processExpiredPayments` expires a payment at `expiryDate` plus `Config.payment.timeoutDelay`, so * the span has to outlast that delay for the expiry transition itself to still be read — which is * why the cutoff below ADDS the configured delay instead of assuming it away. + * + * The hour is chosen against the thing that actually delays the transition: the worker being gone. + * `processExpiredPayments` runs there, so a worker that is down does not expire anything, and the + * transitions arrive in a burst when it returns. Ten minutes covered a deploy; it did not cover an + * outage, and the alert on a silent worker only fires after seventeen. An hour outlasts both, and + * costs a longer read of a handful of rows per connected device. */ -const DEVICE_DELIVERY_GRACE_SECONDS = 600; +const DEVICE_DELIVERY_GRACE_SECONDS = 3600; /** * What this process has delivered to one device: per payment, the wait state last sent for it and @@ -694,46 +704,58 @@ export class PaymentLinkPaymentService { } private async handleQuoteChange(payment: PaymentLinkPayment, quote: PaymentQuote): Promise { - // close activations - if (PaymentQuoteFinalStates.includes(quote.status)) + // Closing the activations of a final quote happens on every path through here, but on ONE of + // them it has to travel with the transition rather than run before it — hence the closure. + // Given a manager it runs inside that transaction; without one it stands alone, as before. + const closeActivations = async (manager?: EntityManager): Promise => { + if (!PaymentQuoteFinalStates.includes(quote.status)) return; + if (payment.mode === PaymentLinkPaymentMode.SINGLE) { - await this.paymentActivationService.closeAllForPayment(payment.id); + await this.paymentActivationService.closeAllForPayment(payment.id, manager); } else { await this.paymentActivationService.closeAllForQuote(quote.id); } + }; - if (payment.status !== PaymentLinkPaymentStatus.PENDING) return; + if (payment.status !== PaymentLinkPaymentStatus.PENDING) return closeActivations(); // update payment status const { minCompletionStatus } = payment.link.configObj; const isPaymentComplete = PaymentQuoteTxStates.indexOf(quote.status) >= PaymentQuoteTxStates.indexOf(minCompletionStatus); - if (isPaymentComplete) { - const txCount = await this.paymentQuoteService.getCompletedQuoteCount(payment, minCompletionStatus); - payment.txCount = txCount; - - // The status read above is the same read-then-write as in expirePayment, and this one is - // reached from request paths as well as from checkTxConfirmations. A `MULTIPLE` payment - // stays `Pending` and has no transition to take: it only counts a quote. - if (payment.mode === PaymentLinkPaymentMode.SINGLE) { - const taken = await this.takePendingTransition( - payment, - PaymentLinkPaymentStatus.COMPLETED, - // The count belongs to the transition: `doSave` below is the only other thing that - // writes it, and no job looks at a completed payment again, so a count left behind by a - // caller that stopped between the two would stay wrong. - async (manager) => { - await manager.update(PaymentLinkPayment, payment.id, { txCount }); - }, - ); - if (!taken) return; - - payment.complete(); - } + if (!isPaymentComplete) return closeActivations(); + + const txCount = await this.paymentQuoteService.getCompletedQuoteCount(payment, minCompletionStatus); + payment.txCount = txCount; + + // The status read above is the same read-then-write as in expirePayment, and this one is + // reached from request paths as well as from checkTxConfirmations. A `MULTIPLE` payment + // stays `Pending` and has no transition to take: it only counts a quote. + if (payment.mode === PaymentLinkPaymentMode.SINGLE) { + const taken = await this.takePendingTransition( + payment, + PaymentLinkPaymentStatus.COMPLETED, + // Both effects belong to the transition. The count, because `doSave` below is the only + // other thing that writes it and no job looks at a completed payment again — a count left + // behind by a caller that stopped between the two would stay wrong. The activations, + // because closing them BEFORE the status moves is the half-state nothing can repair: the + // quote is already final, so `checkTxConfirmations` does not come back to it, while + // `processExpiredPayments` only ever asks for `Pending` and would expire a payment whose + // activations are long closed. + async (manager) => { + await manager.update(PaymentLinkPayment, payment.id, { txCount }); + await closeActivations(manager); + }, + ); + if (!taken) return; - await this.doSave(payment, true); + payment.complete(); + } else { + await closeActivations(); } + + await this.doSave(payment, true); } private async doSave(payment: PaymentLinkPayment, isPaymentDone: boolean): Promise { From 233ba8711d28e0ff967289aed16643376df58090 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:50 +0200 Subject: [PATCH 73/86] Format the added test the way the repo formats everything else --- .../__tests__/monitoring.service.spec.ts | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts b/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts index 96f58b626a..8459429c27 100644 --- a/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts +++ b/src/subdomains/core/monitoring/__tests__/monitoring.service.spec.ts @@ -305,17 +305,15 @@ describe('MonitoringService', () => { }; let idOneExists = false; - const managerFindOne = jest - .fn() - .mockImplementation((_entity: unknown, options: { where?: { id?: number } }) => { - if (options?.where?.id === 1) { - // Missing on the first ask; present once the wait on the old row is over. - return Promise.resolve(idOneExists ? { id: 1, data: JSON.stringify(seededById1) } : null); - } - // The lock on the old row is granted only after the other writer committed. - idOneExists = true; - return Promise.resolve({ id: 7, data: JSON.stringify(persisted) }); - }); + const managerFindOne = jest.fn().mockImplementation((_entity: unknown, options: { where?: { id?: number } }) => { + if (options?.where?.id === 1) { + // Missing on the first ask; present once the wait on the old row is over. + return Promise.resolve(idOneExists ? { id: 1, data: JSON.stringify(seededById1) } : null); + } + // The lock on the old row is granted only after the other writer committed. + idOneExists = true; + return Promise.resolve({ id: 7, data: JSON.stringify(persisted) }); + }); Object.defineProperty(repo, 'manager', { value: { From 1b82229a5d351e8611bb7e5db7291e073ff0dbb0 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:52 +0200 Subject: [PATCH 74/86] Take the kill switch off the bridge, and bound what the record covers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round eight, seven findings, none of them in the lease or the role assignment. The delivery job carried a `process` flag like any other job. It is not a job in that sense: it is the bridge that carries a result from the process that wrote it to the process holding the connection. Off in the single-process setup nothing happens, because `doSave` delivers directly there — so the flag looks harmless right up to the moment it is not. After the split, switching it off silently cuts delivery to everything attached to the other container, and no alert sees it: every process still reports its role and a usable lease. The flag is gone, and with it the enum value this branch had added for it. The same reasoning already governs the role heartbeat and `checkConnections`. The delivery record is written BEFORE the send, and that ordering is now stated rather than implied — together with what it costs and why the alternative costs more. The gateway no longer leaves a socket in the map that threw on `send`: a device unreachable through it would otherwise keep being selected, and counted as delivered. Two comments promised a reconnecting device everything it was owed; they now say "within the span the read covers". Three more: the heartbeat's error form was abbreviated in the comment that declares it an interface, while the alert reads the full line. Two comments still said the alert reports a double run — it reports a wrong ROLE, which is what the logs can distinguish. And the spark job caught its own error to log it again, which is the redundant try-catch CONTRIBUTING names by example. The transaction proof had a hole the review was right to name: the existing tests show the effect services are HANDED the manager, which a service that ignores it would also pass. The new spec asserts the other direction — given a manager the write goes through that manager's repository and never through the injected one, and without one it goes through the injected one. --- docs/cron-jobs.md | 2 +- migration/1785600000000-AddCronLease.js | 7 +- .../blockchain/spark/spark.service.ts | 12 +- src/shared/services/cron-lease.service.ts | 5 +- src/shared/services/dfx-cron.service.ts | 2 +- src/shared/services/process.service.ts | 1 - .../controllers/payment-link.gateway.ts | 35 ++++-- .../__tests__/transactional-effects.spec.ts | 108 ++++++++++++++++++ .../services/payment-cron.service.ts | 17 ++- .../services/payment-link-payment.service.ts | 15 ++- 10 files changed, 174 insertions(+), 30 deletions(-) create mode 100644 src/subdomains/core/payment-link/services/__tests__/transactional-effects.spec.ts diff --git a/docs/cron-jobs.md b/docs/cron-jobs.md index 8a37bd1b19..2a257d8805 100644 --- a/docs/cron-jobs.md +++ b/docs/cron-jobs.md @@ -162,7 +162,7 @@ here rather than fixed in passing. Of the 140 declarations, 139 have a registrat | 10 seconds | `LIQUIDITY_MANAGEMENT` | `worker` | `LiquidityManagementPipelineService::processPipelines` | `subdomains/core/liquidity-management/services/liquidity-management-pipeline.service.ts` | | 10 seconds | `MONITOR_CONNECTION_POOL` | `both` | `MonitorConnectionPoolService::monitorConnectionPoolStatic` | `subdomains/core/monitoring/monitor-connection-pool.service.ts` | | 10 seconds | `MONITOR_EVENT_LOOP` | `both` | `MonitorEventLoopService::monitorEventLoop` | `subdomains/core/monitoring/monitor-event-loop.service.ts` | -| 15 seconds | `PAYMENT_DELIVERY` | `both` | `PaymentCronService::deliverPaymentUpdates` | `subdomains/core/payment-link/services/payment-cron.service.ts` | +| 15 seconds | — | `both` | `PaymentCronService::deliverPaymentUpdates` | `subdomains/core/payment-link/services/payment-cron.service.ts` | | 30 seconds | `LNURL_AUTH_CACHE` | `both` | `AuthLnUrlService::processCleanupAccessToken` | `subdomains/generic/user/models/auth/auth-lnurl.service.ts` | | 30 seconds | — | `both` | `PaymentLinkGateway::checkConnections` | `subdomains/core/payment-link/controllers/payment-link.gateway.ts` | | 30 seconds | `BANK_TX` | `worker` | `BankTxService::checkBankTx` | `subdomains/supporting/bank-tx/bank-tx/services/bank-tx.service.ts` | diff --git a/migration/1785600000000-AddCronLease.js b/migration/1785600000000-AddCronLease.js index 9101b02ade..46841cb3d2 100644 --- a/migration/1785600000000-AddCronLease.js +++ b/migration/1785600000000-AddCronLease.js @@ -9,9 +9,10 @@ * Until now the only thing stopping a job from running twice was `LockClass`, and its state is a * field in process memory — it cannot see a second process. That was acceptable while the API ran * as a single process. With the HTTP process and the worker split apart, "exactly one process runs - * this job" became an assumption held up by configuration, a runbook sentence and an alert that - * *reports* a double run after the fact. For a path that moves money, detection is the second-best - * answer. + * this job" became an assumption held up by configuration, a runbook sentence and an alert — and + * that alert reports a WRONG ROLE, not a double run: it reads the role each process states, which + * says what a process WOULD run, never what two of them did. For a path that moves money, an + * assumption checked from the outside is the second-best answer. * * This table is what a second process has to get past before it may start such a job: a job scoped * to exactly one process must take a row here before it starts, and the row is claimable by one diff --git a/src/integration/blockchain/spark/spark.service.ts b/src/integration/blockchain/spark/spark.service.ts index bb4d0bea6b..02d73dcfe8 100644 --- a/src/integration/blockchain/spark/spark.service.ts +++ b/src/integration/blockchain/spark/spark.service.ts @@ -1,6 +1,5 @@ import { Injectable } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; -import { DfxLogger } from 'src/shared/services/dfx-logger'; import { Process } from 'src/shared/services/process.service'; import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { Bech32mService } from '../shared/bech32m/bech32m.service'; @@ -10,8 +9,6 @@ import { SparkClient, SparkTransaction } from './spark-client'; export class SparkService extends Bech32mService { readonly defaultPrefix = 'spark'; - private readonly logger = new DfxLogger(SparkService); - private readonly client: SparkClient; constructor() { @@ -33,14 +30,13 @@ export class SparkService extends Bech32mService { * includes this work. That is every deployment, for as long as the outgoing container is still * up. Registered here, it goes through the lease like any other worker job. * - * Errors are logged rather than rethrown: this is best-effort housekeeping, the next run is five - * minutes away, and a wallet that cannot be reached now is not a reason to raise an incident. + * Errors are left to the wrapper, per CONTRIBUTING ("@DfxCron already handles errors"). Catching + * them here to log them again was the redundant try-catch that rule names; what it added over + * the wrapper was a lower log level, which is not worth an exception to the rule. */ @DfxCron(CronExpression.EVERY_5_MINUTES, { scope: CronScope.WORKER, process: Process.SPARK_TOKEN_OPTIMIZATION }) async optimizeTokenOutputs(): Promise { - await this.client - .optimizeTokenOutputs() - .catch((e) => this.logger.warn('Token optimization failed, will retry on the next run:', e)); + await this.client.optimizeTokenOutputs(); } async isHealthy(): Promise { diff --git a/src/shared/services/cron-lease.service.ts b/src/shared/services/cron-lease.service.ts index 60f8ac68a2..f8f4193d2c 100644 --- a/src/shared/services/cron-lease.service.ts +++ b/src/shared/services/cron-lease.service.ts @@ -47,8 +47,9 @@ const SHUTDOWN_GRACE_MS = 10 * 1000; * `LockClass` keeps its state in a field of a process-local object. That was enough while the API * ran as one process; it cannot see a second one. Since the HTTP process and the worker are split * apart, "exactly one process runs this job" rests on configuration, a runbook sentence and an - * alert that *reports* a double run after the fact. For a path that moves money that is the - * second-best answer, so this adds a layer underneath it. + * alert — and that alert reports a WRONG ROLE, not a double run: a role that both processes can + * see is not one the logs distinguish. For a path that moves money, an assumption checked from + * the outside is the second-best answer, so this adds a layer underneath it. * * A layer, not a guarantee — read "What it does not do" below before relying on this. A job runs * once because the deployment runs one worker and because the job tolerates being run again; what diff --git a/src/shared/services/dfx-cron.service.ts b/src/shared/services/dfx-cron.service.ts index 5ff7a49256..3fcf8aa202 100644 --- a/src/shared/services/dfx-cron.service.ts +++ b/src/shared/services/dfx-cron.service.ts @@ -95,7 +95,7 @@ export class DfxCronService implements OnModuleInit { * * ``` * CronRole : heartbeat, jobs registered, lease ok - * CronRole : heartbeat, jobs registered, lease unusable: + * CronRole : heartbeat, jobs registered, lease unusable: failure(s) since the last heartbeat, last error: * ``` * * Three properties make that safe to match on, and all three are load-bearing. One of `lease ok` diff --git a/src/shared/services/process.service.ts b/src/shared/services/process.service.ts index d262fb45f3..d00d31deaa 100644 --- a/src/shared/services/process.service.ts +++ b/src/shared/services/process.service.ts @@ -62,7 +62,6 @@ export enum Process { SCORECHAIN = 'Scorechain', PAYMENT_EXPIRATION = 'PaymentExpiration', PAYMENT_CONFIRMATIONS = 'PaymentConfirmations', - PAYMENT_DELIVERY = 'PaymentDelivery', PAYMENT_FORWARDING = 'PaymentForwarding', FIAT_OUTPUT = 'FiatOutput', FIAT_OUTPUT_ASSIGN_BANK_ACCOUNT = 'FiatOutputAssignBankAccount', diff --git a/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts b/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts index 19082433f4..8b6e8a0a42 100644 --- a/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts +++ b/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts @@ -33,7 +33,7 @@ export class PaymentLinkGateway implements OnGatewayConnection, OnModuleInit { constructor(private readonly paymentService: PaymentLinkPaymentService) {} - onModuleInit() { + onModuleInit(): void { this.paymentService.getDeviceActivationObservable().subscribe((a) => this.sendMessage(a)); // The delivery reads what is connected out of the map below instead of being told about it, so @@ -41,7 +41,7 @@ export class PaymentLinkGateway implements OnGatewayConnection, OnModuleInit { this.paymentService.useDeviceSource(() => this.connectedDevices()); } - handleConnection(client: PaymentSocket, message: IncomingMessage) { + handleConnection(client: PaymentSocket, message: IncomingMessage): void { const device = new URLSearchParams(message.url?.split('?')[1]).get('device'); if (!device) throw new BadRequestException('device should not be empty'); @@ -53,7 +53,9 @@ export class PaymentLinkGateway implements OnGatewayConnection, OnModuleInit { * * A device appears here for exactly as long as at least one of its connections is in the map. * When it connected is deliberately not part of it: the delivery selects payments by their own - * lifetime, so a device that reconnects is owed the same thing it was owed before. + * lifetime, so a device that reconnects is owed what it was owed before — for as long as those + * payments are still inside the read (see DEVICE_DELIVERY_GRACE_SECONDS). A device that stays + * away past that gets nothing for the payments that aged out meanwhile. */ connectedDevices(): ConnectedDevice[] { return [...this.clients.keys()].map((id) => ({ id })); @@ -93,7 +95,7 @@ export class PaymentLinkGateway implements OnGatewayConnection, OnModuleInit { } // --- HELPER METHODS --- // - private addClient(device: string, client: PaymentSocket) { + private addClient(device: string, client: PaymentSocket): void { const clientId = Util.createUniqueId('client'); const connections = this.clients.get(device) ?? new Map(); @@ -107,7 +109,7 @@ export class PaymentLinkGateway implements OnGatewayConnection, OnModuleInit { client.on('pong', () => this.markResponsive(device, clientId)); } - private removeClient(device: string, clientId: string) { + private removeClient(device: string, clientId: string): void { const connections = this.clients.get(device); if (!connections) return; @@ -118,17 +120,32 @@ export class PaymentLinkGateway implements OnGatewayConnection, OnModuleInit { if (!connections.size) this.clients.delete(device); } - private markResponsive(device: string, clientId: string) { + private markResponsive(device: string, clientId: string): void { const connection = this.clients.get(device)?.get(clientId); if (connection) connection.responsive = true; } - private sendMessage(device: PaymentDevice) { + /** + * Sends to every socket of a device, and drops the ones that cannot take it. + * + * A `send` that throws leaves a socket the peer is no longer on. Left in the map it would keep + * the device in `connectedDevices`, so the delivery would go on selecting payments for a device + * it can no longer reach — and would count them as delivered. Removal is idempotent and the + * `close`/`error` handlers do the same thing, so a socket that reports both is removed once. + * + * One failing socket does not stop the others: a device with two connections is still reachable + * through the second. + */ + private sendMessage(device: PaymentDevice): void { const connections = this.clients.get(device.id); if (!connections) return; - for (const { socket } of connections.values()) { - socket.send(device.command); + for (const [clientId, { socket }] of [...connections]) { + try { + socket.send(device.command); + } catch { + this.removeClient(device.id, clientId); + } } } } diff --git a/src/subdomains/core/payment-link/services/__tests__/transactional-effects.spec.ts b/src/subdomains/core/payment-link/services/__tests__/transactional-effects.spec.ts new file mode 100644 index 0000000000..25f43559c6 --- /dev/null +++ b/src/subdomains/core/payment-link/services/__tests__/transactional-effects.spec.ts @@ -0,0 +1,108 @@ +import { EntityManager } from 'typeorm'; +import { PaymentActivation } from '../../entities/payment-activation.entity'; +import { PaymentActivationService } from '../payment-activation.service'; +import { PaymentQuoteService } from '../payment-quote.service'; + +/** + * The other half of the transaction proof. + * + * `payment-link-payment.service.spec.ts` pins that the transitions HAND their effect services the + * manager they were given. That alone proves nothing about where the statement lands: a service + * that accepts the argument and then writes through its own repository would pass that test and + * still commit on its own — which is exactly the half-state the transaction exists to rule out, + * and the review that asked for this test was right that nothing here ruled it out. + * + * So this side asserts the opposite direction: given a manager, the effect goes through THAT + * manager's repository and never through the injected one. Given none, it goes through the + * injected one — the paths outside a transition still work. + */ +describe('effects that run inside a caller transaction', () => { + type RepoMock = { update: jest.Mock; find: jest.Mock; save: jest.Mock }; + + let injectedRepo: RepoMock; + let managerRepo: RepoMock; + let manager: EntityManager; + + /** The write surfaces both effect services use; `find` answers with one row so `save` runs. */ + function repoMock(): RepoMock { + return { + update: jest.fn().mockResolvedValue({ affected: 1 }), + find: jest.fn().mockResolvedValue([{ cancel: () => ({ id: 1 }) }]), + save: jest.fn().mockResolvedValue(undefined), + }; + } + + /** Every way a repository is written through, so a test cannot miss one by naming the wrong. */ + function writes(repo: RepoMock): number { + return repo.update.mock.calls.length + repo.save.mock.calls.length; + } + + beforeEach(() => { + injectedRepo = repoMock(); + managerRepo = repoMock(); + manager = { getRepository: jest.fn().mockReturnValue(managerRepo) } as unknown as EntityManager; + }); + + describe('PaymentActivationService.closeAllForPayment', () => { + function service(): PaymentActivationService { + // Only the repository and the one collaborator the constructor calls are real; the rest is + // untouched by the path under test. + const lightningService = { getDefaultClient: () => undefined } as never; + + const u = undefined as never; + + return new PaymentActivationService(lightningService, injectedRepo as unknown as never, u, u, u, u, u); + } + + it('writes through the manager it was given, not through its own repository', async () => { + await service().closeAllForPayment(7, manager); + + expect(manager.getRepository).toHaveBeenCalledWith(PaymentActivation); + expect(writes(managerRepo)).toEqual(1); + // The one that would commit on its own. + expect(writes(injectedRepo)).toEqual(0); + }); + + it('falls back to its own repository when there is no transaction to join', async () => { + await service().closeAllForPayment(7); + + expect(writes(injectedRepo)).toEqual(1); + expect(writes(managerRepo)).toEqual(0); + }); + + it('closes exactly the activations of that payment that are still open', async () => { + // The criteria matter as much as the connection: closing an already closed activation is + // harmless, closing another payment's is not. + await service().closeAllForPayment(7, manager); + + const [criteria] = managerRepo.update.mock.calls[0]; + + expect(criteria.payment).toEqual({ id: 7 }); + expect(criteria.status).toBeDefined(); + }); + }); + + describe('PaymentQuoteService.cancelAllForPayment', () => { + function service(): PaymentQuoteService { + const u = undefined as never; + + return new PaymentQuoteService(injectedRepo as unknown as never, u, u, u, u, u, u, u, u); + } + + it('writes through the manager it was given, not through its own repository', async () => { + await service().cancelAllForPayment(7, manager); + + expect(managerRepo.find).toHaveBeenCalledTimes(1); + expect(writes(managerRepo)).toEqual(1); + expect(injectedRepo.find).not.toHaveBeenCalled(); + expect(writes(injectedRepo)).toEqual(0); + }); + + it('falls back to its own repository when there is no transaction to join', async () => { + await service().cancelAllForPayment(7); + + expect(writes(injectedRepo)).toEqual(1); + expect(writes(managerRepo)).toEqual(0); + }); + }); +}); diff --git a/src/subdomains/core/payment-link/services/payment-cron.service.ts b/src/subdomains/core/payment-link/services/payment-cron.service.ts index 79d63cc5f6..598d4f90cb 100644 --- a/src/subdomains/core/payment-link/services/payment-cron.service.ts +++ b/src/subdomains/core/payment-link/services/payment-cron.service.ts @@ -48,12 +48,21 @@ export class PaymentCronService { // `useDelay: false` for the same reason: the jitter exists to spread jobs that do real work per // run, and up to five seconds of it would be a third of this interval. // - // Switching the flag off does not stop payments from being processed, but callers of - // `GET /v1/paymentLink/payment/wait` and `GET /v1/lnurlp/wait/:id` on a process that is not the - // one writing then stay connected until they give up. + // Deliberately WITHOUT a `process` flag, and that is a correction. It carried one, and the flag + // looked like any other job's — but this job is not work, it is the bridge that carries a result + // from the process that wrote it to the process holding the connection. Switched off in the + // single-process setup nothing happens, because `doSave` delivers directly there; switched off + // after the split it silently cuts delivery to everything attached to the OTHER container: + // waiting callers of `GET /v1/paymentLink/payment/wait` and `GET /v1/lnurlp/wait/:id` hang until + // they give up, and connected devices are never told their payment went through. No alert sees + // it — every process still reports its role and a usable lease. + // + // A switch whose failure mode is invisible is worse than no switch. The same reasoning already + // applies to the role heartbeat and to `PaymentLinkGateway.checkConnections`: a mechanism the + // rest depends on does not get a kill switch. Whatever a switch here would have been used for — + // load, a misbehaving device — is reached by disabling the jobs that WRITE, which do have flags. @DfxCron(CustomCronExpression.EVERY_15_SECONDS, { scope: CronScope.BOTH, - process: Process.PAYMENT_DELIVERY, useDelay: false, }) async deliverPaymentUpdates(): Promise { diff --git a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts index 71f59081e4..bad9454895 100644 --- a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts @@ -51,7 +51,8 @@ const PAYMENT_WAIT_TIMEOUT_SECONDS = 60; * * Only the identity: what a device is owed follows from the payments themselves, not from when it * happened to connect. A device that reconnects is the same device, and the delivery record below - * outlives the connection precisely so that it is treated as one. + * outlives the connection precisely so that it is treated as one — within the span the read + * covers. What has aged out of that span is owed to nobody any more. */ export interface ConnectedDevice { id: string; @@ -445,6 +446,18 @@ export class PaymentLinkPaymentService { const delivered = this.deliveriesFor(connected.id); if (delivered.get(payment.id)?.state === payment.waitState) return; + // Recorded BEFORE the send, and the two ways that can be wrong are not equal. A send that + // fails after this line is not retried, because the record says it went out; a record written + // after a successful send would instead repeat every command whenever the send path is slower + // than the next tick. The first costs one missed command on a socket that is being torn down — + // and the gateway drops such a socket, so the device stops being connected and the command + // goes out again on the next tick under a new connection. The second costs a repeat on a + // healthy one. + // + // What no ordering fixes: this record lives in the process. A restart empties it, and a device + // still inside the read gets its current state again. Delivering the same state twice is what + // `waitState` makes harmless; the alternative — persisting a delivery log — is a table for a + // problem a duplicate command already answers. delivered.set(payment.id, { state: payment.waitState, expiryDate: payment.expiryDate }); this.deviceActivationSubject.next(device); From 05f5719c900445c13f756d1a3ff9a1bf77be5906 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:53 +0200 Subject: [PATCH 75/86] Record a device command once it is out, not once it is handed over MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round nine found the delivery could lose a command silently, and the code said the opposite in two places that could not both be true. The record of what a device has been told is keyed by DEVICE and outlives the connection — `pruneDeliveries` keeps it that way on purpose, so a reconnecting device is not told twice. The write path meanwhile claimed that a failed send would go out again "on the next tick under a new connection". It would not: the record already said delivered, and the periodic delivery that would repair it takes the early return. A `send` that threw was therefore lost for good, on the one path built to prevent exactly that. The gateway now hands the command over through a sink that answers whether it reached a socket, and the record is written only on `true`. That is what the RxJS subject could not do: a subject carries a value one way and swallows what the subscriber does with it, including a throw. It had one consumer, so it is gone rather than wrapped. The cost of this order is a repeat if the process dies between the send and the record. Repeating is what `waitState` makes harmless; losing is not. Also: `docs/cron-jobs.md` counted 118 of 140 jobs as flagged with six deliberate exceptions. Removing the delivery job's flag last round made that 117, 23 and seven — recounted from the table rather than adjusted by one, and the delivery job now appears in the list of deliberate exceptions with its reason instead of silently among the omissions. --- docs/cron-jobs.md | 13 ++-- .../__tests__/payment-link.gateway.spec.ts | 46 +++++++++++-- .../controllers/payment-link.gateway.ts | 26 ++++++-- .../payment-link-payment.service.spec.ts | 64 ++++++++++++++++++- .../services/payment-link-payment.service.ts | 46 ++++++++----- 5 files changed, 159 insertions(+), 36 deletions(-) diff --git a/docs/cron-jobs.md b/docs/cron-jobs.md index 2a257d8805..ad47fae58b 100644 --- a/docs/cron-jobs.md +++ b/docs/cron-jobs.md @@ -31,19 +31,22 @@ request path loads on demand, and a job may refresh it but must not be the only ## Flags -118 of the 140 jobs carry a `process` flag, 22 do not. A job with a flag can be switched off +117 of the 140 jobs carry a `process` flag, 23 do not. A job with a flag can be switched off without a deploy — `DfxCronService` skips it when the process appears in the disabled set, which `ProcessService` refreshes from the `disabledProcesses` setting and the `DISABLED_PROCESSES` environment variable every 30 seconds. -A job **without** a flag runs unconditionally. That is deliberate for six of them. The four +A job **without** a flag runs unconditionally. That is deliberate for seven of them. The four `ProcessService::resync*` jobs maintain the disabled set, the JWT denylists and the staff clearance allowlist themselves, so making them switchable would let a configuration change disable the mechanism that reads configuration changes. `DfxCronService::reportRole` is the role heartbeat: switched off, it would look exactly like a process that stopped reporting, which is the condition it exists to make visible. `PaymentLinkGateway::checkConnections` drops websockets -that stopped answering, so switching it off would reinstate the unbounded growth it prevents. For -the remaining 16 it is simply an omission: +that stopped answering, so switching it off would reinstate the unbounded growth it prevents. +`PaymentCronService::deliverPaymentUpdates` is the bridge between the process that writes a +payment and the one holding the connection: switched off it changes nothing in a single-process +setup, because `doSave` delivers directly there, and after the split it silently cuts delivery to +everything attached to the other container. For the remaining 16 it is simply an omission: | Job | Interval | | --- | --- | @@ -93,7 +96,7 @@ Jobs by area: | `shared/services` | 5 | 5 | | `subdomains/core/sell-crypto` | 5 | 2 | | `subdomains/supporting/payment` | 5 | 1 | -| `subdomains/core/payment-link` | 6 | 1 | +| `subdomains/core/payment-link` | 6 | 2 | | `subdomains/generic/kyc` | 4 | — | | `subdomains/supporting/bank` | 4 | — | | `subdomains/supporting/bank-tx` | 4 | — | diff --git a/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts b/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts index b257b48fef..4184d9f2f8 100644 --- a/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts +++ b/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts @@ -1,5 +1,4 @@ import { IncomingMessage } from 'http'; -import { Subject } from 'rxjs'; import { CronScope, DFX_CRONJOB_PARAMS, DfxCronParams } from 'src/shared/utils/cron'; import { PaymentDevice } from '../../entities/payment-link-payment.entity'; import { PaymentLinkPaymentService } from '../../services/payment-link-payment.service'; @@ -13,7 +12,6 @@ import { PaymentLinkGateway } from '../payment-link.gateway'; describe('PaymentLinkGateway', () => { let gateway: PaymentLinkGateway; let paymentService: jest.Mocked; - let activations: Subject; /** A socket that records what was done to it and lets a test fire its events. */ function socket() { @@ -51,10 +49,12 @@ describe('PaymentLinkGateway', () => { const deviceIds = () => gateway.connectedDevices().map((d) => d.id); + /** The sink the gateway registers on init; calling it is what the delivery does. */ + let deliver: (device: PaymentDevice) => boolean; + beforeEach(() => { - activations = new Subject(); paymentService = { - getDeviceActivationObservable: () => activations.asObservable(), + useDeviceSink: jest.fn().mockImplementation((sink) => (deliver = sink)), useDeviceSource: jest.fn(), } as unknown as jest.Mocked; @@ -204,10 +204,46 @@ describe('PaymentLinkGateway', () => { const second = connect('pos-1'); const other = connect('pos-2'); - activations.next({ id: 'pos-1', command: 'show-paid' }); + expect(deliver({ id: 'pos-1', command: 'show-paid' })).toBe(true); expect(first.sent).toEqual(['show-paid']); expect(second.sent).toEqual(['show-paid']); expect(other.sent).toEqual([]); }); + + it('reports nothing delivered when the device has no connection here', () => { + // `false` is what keeps the delivery from recording a command it never sent — a device + // connected to the OTHER process must stay owed. + gateway.onModuleInit(); + + expect(deliver({ id: 'pos-unknown', command: 'show-paid' })).toBe(false); + }); + + it('reports nothing delivered when every socket of the device throws, and drops them', () => { + gateway.onModuleInit(); + + const client = connect('pos-1'); + client.send = () => { + throw new Error('socket closed'); + }; + + expect(deliver({ id: 'pos-1', command: 'show-paid' })).toBe(false); + // Dropped, so the device stops being selected for delivery at all. + expect(deviceIds()).toEqual([]); + }); + + it('reports delivered when one socket takes it and another throws', () => { + gateway.onModuleInit(); + + const broken = connect('pos-1'); + broken.send = () => { + throw new Error('socket closed'); + }; + const working = connect('pos-1'); + + expect(deliver({ id: 'pos-1', command: 'show-paid' })).toBe(true); + expect(working.sent).toEqual(['show-paid']); + // The device is still reachable through the one that worked. + expect(deviceIds()).toEqual(['pos-1']); + }); }); diff --git a/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts b/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts index 8b6e8a0a42..9b3ff261b5 100644 --- a/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts +++ b/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts @@ -34,7 +34,9 @@ export class PaymentLinkGateway implements OnGatewayConnection, OnModuleInit { constructor(private readonly paymentService: PaymentLinkPaymentService) {} onModuleInit(): void { - this.paymentService.getDeviceActivationObservable().subscribe((a) => this.sendMessage(a)); + // Handed over rather than subscribed to: the delivery has to know whether the command reached + // a socket, and that is what `sendMessage` answers. A subscription could not say. + this.paymentService.useDeviceSink((device) => this.sendMessage(device)); // The delivery reads what is connected out of the map below instead of being told about it, so // there is no second register to keep in step. See PaymentLinkPaymentService.connectedDevices. @@ -126,26 +128,36 @@ export class PaymentLinkGateway implements OnGatewayConnection, OnModuleInit { } /** - * Sends to every socket of a device, and drops the ones that cannot take it. + * Sends to every socket of a device, drops the ones that cannot take it, and reports whether + * any of them took it. + * + * The return value is what the delivery records against: it only marks a command as delivered + * once one actually left. `false` therefore has to mean "nothing got out" — for a device with + * no connection here at all, and for one whose every socket threw. * * A `send` that throws leaves a socket the peer is no longer on. Left in the map it would keep * the device in `connectedDevices`, so the delivery would go on selecting payments for a device - * it can no longer reach — and would count them as delivered. Removal is idempotent and the - * `close`/`error` handlers do the same thing, so a socket that reports both is removed once. + * it can no longer reach. Removal is idempotent and the `close`/`error` handlers do the same + * thing, so a socket that reports both is removed once. * * One failing socket does not stop the others: a device with two connections is still reachable - * through the second. + * through the second, and that counts as delivered. */ - private sendMessage(device: PaymentDevice): void { + private sendMessage(device: PaymentDevice): boolean { const connections = this.clients.get(device.id); - if (!connections) return; + if (!connections) return false; + + let delivered = false; for (const [clientId, { socket }] of [...connections]) { try { socket.send(device.command); + delivered = true; } catch { this.removeClient(device.id, clientId); } } + + return delivered; } } diff --git a/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts b/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts index 0f6cc70aae..1471cab6ca 100644 --- a/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts +++ b/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts @@ -53,9 +53,17 @@ describe('PaymentLinkPaymentService', () => { ]); } - function devices(): PaymentDevice[] { + /** + * Stands in for the gateway's sockets. Records what was handed over AND answers whether it got + * out — the delivery only marks a command as delivered on `true`, so a sink that always said + * yes would hide exactly the failure the record has to survive. + */ + function devices(delivers = true): PaymentDevice[] { const seen: PaymentDevice[] = []; - service.getDeviceActivationObservable().subscribe((device) => seen.push(device)); + service.useDeviceSink((device) => { + seen.push(device); + return delivers; + }); return seen; } @@ -438,12 +446,64 @@ describe('PaymentLinkPaymentService', () => { expect(seen).toHaveLength(2); }); + it('should keep owing a command whose send did not get out', async () => { + // The loss this closes: the record is keyed by DEVICE and outlives the connection, so a + // command marked delivered before a failing send would never be retried — not even when + // the device reconnects, because the record still says it heard this state. Recording only + // what got out is what keeps the periodic delivery able to repair it. + const seen = devices(false); + connect('pos-1'); + paymentLinkPaymentRepo.find.mockImplementation(async (options) => findByCutoff(options)); + + rows = [ + payment({ + id: 7, + status: PaymentLinkPaymentStatus.COMPLETED, + deviceId: 'pos-1', + deviceCommand: 'show-paid', + }), + ]; + + await service.deliverPaymentUpdates(); + expect(seen).toHaveLength(1); + // Nothing got out, so nothing is recorded. + expect(service['deviceDeliveries'].get('pos-1')?.size ?? 0).toEqual(0); + + // The next tick tries again — which is the whole point. + await service.deliverPaymentUpdates(); + expect(seen).toHaveLength(2); + }); + + it('should stop repeating once a command does get out', async () => { + // The other direction: a sink that takes it must end the repetition, or the fix above would + // have traded a silent loss for an endless one. + const seen = devices(); + connect('pos-1'); + paymentLinkPaymentRepo.find.mockImplementation(async (options) => findByCutoff(options)); + + rows = [ + payment({ + id: 7, + status: PaymentLinkPaymentStatus.COMPLETED, + deviceId: 'pos-1', + deviceCommand: 'show-paid', + }), + ]; + + await service.deliverPaymentUpdates(); + await service.deliverPaymentUpdates(); + + expect(seen).toHaveLength(1); + }); + it('should forget a payment the read has left behind, and the device with its last one', async () => { // What bounds the record is the same cutoff the query uses: a payment the read can no longer // return cannot be delivered again, so nothing is kept for it. Without that, a device // connected all day would accumulate an entry per payment it ever saw — and a device that // never comes back would keep a map of its own for good. connect('pos-1'); + // A sink that takes it: the record is only written for a command that got out. + devices(); paymentLinkPaymentRepo.find.mockImplementation(async (options) => findByCutoff(options)); // Delivered directly by the writing process, which does not consult the cutoff. diff --git a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts index bad9454895..24a28c7c67 100644 --- a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts @@ -1,5 +1,4 @@ import { BadRequestException, ConflictException, Injectable, NotFoundException } from '@nestjs/common'; -import { Observable, Subject } from 'rxjs'; import { Config, Environment } from 'src/config/config'; import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.enum'; import { BlockchainRegistryService } from 'src/integration/blockchain/shared/services/blockchain-registry.service'; @@ -99,7 +98,11 @@ type DeviceDeliveries = Map; export class PaymentLinkPaymentService { private readonly paymentWaitMap = new AsyncMap(this.constructor.name); private readonly waitStates = new Map(); - private readonly deviceActivationSubject = new Subject(); + /** + * Where a device command is HANDED OVER, and what says whether it got there. Empty until the + * gateway sets it, which is the honest answer for a process holding no websocket at all. + */ + private deviceSink: (device: PaymentDevice) => boolean = () => false; private readonly deviceDeliveries = new Map(); /** @@ -127,8 +130,17 @@ export class PaymentLinkPaymentService { private readonly blockchainRegistryService: BlockchainRegistryService, ) {} - getDeviceActivationObservable(): Observable { - return this.deviceActivationSubject.asObservable(); + /** + * Points the delivery at the gateway's sockets; see `deliverToDevice` for why it returns a + * boolean. Called once, by PaymentLinkGateway, which is the only thing that holds them. + * + * Replaces an RxJS subject the gateway subscribed to. A subject carries a value one way and + * swallows what the subscriber does with it — including a `send` that threw — and this delivery + * has to know whether the command arrived: the record it keeps is a record of what a device HAS + * been told. A subject cannot answer that question, so it was the wrong shape here. + */ + useDeviceSink(sink: (device: PaymentDevice) => boolean): void { + this.deviceSink = sink; } // --- JOBS --- // @@ -298,7 +310,7 @@ export class PaymentLinkPaymentService { /** * Both delivery channels of this service are process-local: the map behind this method and the - * subject behind `getDeviceActivationObservable`. A caller therefore only ever hears from the + * sink behind `useDeviceSink`. A caller therefore only ever hears from the * process holding its connection, while the jobs that move a payment forward run in one process * (`CronScope.WORKER`, which the deployment runs once and the cron lease keeps to one claim). * @@ -446,21 +458,21 @@ export class PaymentLinkPaymentService { const delivered = this.deliveriesFor(connected.id); if (delivered.get(payment.id)?.state === payment.waitState) return; - // Recorded BEFORE the send, and the two ways that can be wrong are not equal. A send that - // fails after this line is not retried, because the record says it went out; a record written - // after a successful send would instead repeat every command whenever the send path is slower - // than the next tick. The first costs one missed command on a socket that is being torn down — - // and the gateway drops such a socket, so the device stops being connected and the command - // goes out again on the next tick under a new connection. The second costs a repeat on a - // healthy one. + // Recorded only once the command is out, and that ordering is the whole point of the sink's + // return value. Recording first was wrong in a way the comment here used to paper over: it + // claimed a failed send would go out again "on the next tick under a new connection", but the + // record is keyed by DEVICE, not by connection, and `pruneDeliveries` keeps it precisely so a + // reconnecting device is not told twice. A send that threw would therefore have been recorded + // as delivered and never retried — a silent loss on the one path that exists to prevent one. + // + // The cost of this order is a repeat when the command arrives but the process dies before the + // record is written. Repeating is what `waitState` makes harmless; losing is not. // // What no ordering fixes: this record lives in the process. A restart empties it, and a device - // still inside the read gets its current state again. Delivering the same state twice is what - // `waitState` makes harmless; the alternative — persisting a delivery log — is a table for a - // problem a duplicate command already answers. - delivered.set(payment.id, { state: payment.waitState, expiryDate: payment.expiryDate }); + // still inside the read gets its current state again — the same harmless repeat. + if (!this.deviceSink(device)) return; - this.deviceActivationSubject.next(device); + delivered.set(payment.id, { state: payment.waitState, expiryDate: payment.expiryDate }); } async handleBinanceWaiting(result: C2BWebhookResult): Promise { From ed38677e615cfb5bc1a04cbe962ec678a8c91901 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:55 +0200 Subject: [PATCH 76/86] Ask the socket whether it is open, because send will not say MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round nine moved the delivery record behind the send so a failed one would be retried. Round ten found the half of that which `send` cannot report. `ws.send()` throws only while the socket is still CONNECTING. On one that is CLOSING or CLOSED it takes the call without a word and raises the failure asynchronously through `'error'` — by which time the sink has already answered `true` and the delivery has recorded the command against a state it will not send again, not even when the device reconnects. That is the same silent loss as before, reached through the one door the try/catch does not cover. So the state is read first and only an open socket is written to; one that is not open is dropped, exactly like one that throws. What remains is a socket closing BETWEEN the check and the send — one synchronous call wide, and the delivery survives that the way it survives a restart, because the payment stays inside the read until its own end. What it does not survive is being told `true` for a socket that was already gone. Three smaller ones. Two comments still described the RxJS subject that round nine replaced — a later reader would have looked for a subscription instead of the sink and missed that it answers. `PaymentLinkFeeService` justified its `Api` scope with "one of the Api-scoped payment crons"; it is the only one in the domain, and it writes the cache rather than reading it. And two imports were not alphabetical. --- .../spark/__tests__/spark.service.spec.ts | 2 +- .../__tests__/payment-link.gateway.spec.ts | 30 ++ .../controllers/payment-link.gateway.ts | 31 ++- .../__tests__/payment-cron.service.spec.ts | 2 +- .../services/payment-cron.service.ts | 152 +++++----- .../services/payment-link-fee.service.ts | 259 +++++++++--------- .../services/payment-link-payment.service.ts | 7 +- 7 files changed, 271 insertions(+), 212 deletions(-) diff --git a/src/integration/blockchain/spark/__tests__/spark.service.spec.ts b/src/integration/blockchain/spark/__tests__/spark.service.spec.ts index b29113a564..133b460788 100644 --- a/src/integration/blockchain/spark/__tests__/spark.service.spec.ts +++ b/src/integration/blockchain/spark/__tests__/spark.service.spec.ts @@ -1,4 +1,4 @@ -import { DFX_CRONJOB_PARAMS, CronScope, DfxCronParams } from 'src/shared/utils/cron'; +import { CronScope, DFX_CRONJOB_PARAMS, DfxCronParams } from 'src/shared/utils/cron'; import { SparkService } from '../spark.service'; jest.mock('@buildonspark/spark-sdk', () => ({ diff --git a/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts b/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts index 4184d9f2f8..419a240d8f 100644 --- a/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts +++ b/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts @@ -21,6 +21,8 @@ describe('PaymentLinkGateway', () => { sent: [] as string[], pings: 0, terminated: false, + // `ws.OPEN`. A test that wants a closing socket sets it to 2 (`CLOSING`) or 3 (`CLOSED`). + readyState: 1 as 0 | 1 | 2 | 3, send(data: string) { this.sent.push(data); }, @@ -219,6 +221,34 @@ describe('PaymentLinkGateway', () => { expect(deliver({ id: 'pos-unknown', command: 'show-paid' })).toBe(false); }); + it('reports nothing delivered when the socket is closing, and drops it', () => { + // The path `send` cannot report: on a CLOSING or CLOSED socket `ws` takes the call quietly + // and raises the failure through `'error'` later. Without reading the state first the sink + // would answer `true`, and the delivery would record a command that never went out — against + // a state it will not send again, not even when the device reconnects. + gateway.onModuleInit(); + + const client = connect('pos-1'); + client.readyState = 2; + + expect(deliver({ id: 'pos-1', command: 'show-paid' })).toBe(false); + expect(client.sent).toEqual([]); + expect(deviceIds()).toEqual([]); + }); + + it('reports delivered when one socket is closing and another is open', () => { + gateway.onModuleInit(); + + const closing = connect('pos-1'); + closing.readyState = 3; + const open = connect('pos-1'); + + expect(deliver({ id: 'pos-1', command: 'show-paid' })).toBe(true); + expect(closing.sent).toEqual([]); + expect(open.sent).toEqual(['show-paid']); + expect(deviceIds()).toEqual(['pos-1']); + }); + it('reports nothing delivered when every socket of the device throws, and drops them', () => { gateway.onModuleInit(); diff --git a/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts b/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts index 9b3ff261b5..77ecc7400c 100644 --- a/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts +++ b/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts @@ -14,12 +14,22 @@ import { ConnectedDevice, PaymentLinkPaymentService } from '../services/payment- * so its behaviour can be exercised without standing up a server. */ interface PaymentSocket { + /** + * `ws`'s connection state. Read before sending, because `send` is not the place a closing + * socket reports itself: it throws only while still `CONNECTING`, and on a socket that is + * `CLOSING` or `CLOSED` it returns quietly and raises the failure through `'error'` — long + * after the caller decided the command was out. See `sendMessage`. + */ + readonly readyState: 0 | 1 | 2 | 3; send(data: string): void; ping(): void; terminate(): void; on(event: 'close' | 'error' | 'pong', listener: () => void): void; } +/** `ws.OPEN`. Named rather than imported, so the interface above stays the whole dependency. */ +const SOCKET_OPEN = 1; + /** One open websocket, and what is known about it here. */ interface Connection { socket: PaymentSocket; @@ -133,15 +143,27 @@ export class PaymentLinkGateway implements OnGatewayConnection, OnModuleInit { * * The return value is what the delivery records against: it only marks a command as delivered * once one actually left. `false` therefore has to mean "nothing got out" — for a device with - * no connection here at all, and for one whose every socket threw. + * no connection here at all, and for one whose sockets could not take it. + * + * TWO checks, because `send` alone does not tell the difference. It throws while the socket is + * still `CONNECTING`, but a socket that is `CLOSING` or `CLOSED` takes the call without a word + * and reports the failure asynchronously through `'error'` — by which time the caller has + * already been told the command went out, and the delivery has recorded it against a state it + * will not send again. So the state is read first, and only an open socket is written to. * - * A `send` that throws leaves a socket the peer is no longer on. Left in the map it would keep + * A socket that fails either way is one the peer is no longer on. Left in the map it would keep * the device in `connectedDevices`, so the delivery would go on selecting payments for a device * it can no longer reach. Removal is idempotent and the `close`/`error` handlers do the same * thing, so a socket that reports both is removed once. * * One failing socket does not stop the others: a device with two connections is still reachable * through the second, and that counts as delivered. + * + * What remains uncovered: a socket that closes BETWEEN the check and the send. That window is + * the width of one synchronous call, and the delivery survives it the same way it survives a + * process restart — the payment stays inside the read until its own end, so a later tick tries + * again as soon as the device reconnects and the record has aged out. What it does not survive + * is being told `true` for a socket that was already gone, which is what this closes. */ private sendMessage(device: PaymentDevice): boolean { const connections = this.clients.get(device.id); @@ -150,6 +172,11 @@ export class PaymentLinkGateway implements OnGatewayConnection, OnModuleInit { let delivered = false; for (const [clientId, { socket }] of [...connections]) { + if (socket.readyState !== SOCKET_OPEN) { + this.removeClient(device.id, clientId); + continue; + } + try { socket.send(device.command); delivered = true; diff --git a/src/subdomains/core/payment-link/services/__tests__/payment-cron.service.spec.ts b/src/subdomains/core/payment-link/services/__tests__/payment-cron.service.spec.ts index 0884cf1e65..782fcbcde7 100644 --- a/src/subdomains/core/payment-link/services/__tests__/payment-cron.service.spec.ts +++ b/src/subdomains/core/payment-link/services/__tests__/payment-cron.service.spec.ts @@ -1,4 +1,4 @@ -import { DFX_CRONJOB_PARAMS, CronScope, DfxCronParams } from 'src/shared/utils/cron'; +import { CronScope, DFX_CRONJOB_PARAMS, DfxCronParams } from 'src/shared/utils/cron'; import { PaymentCronService } from '../payment-cron.service'; /** diff --git a/src/subdomains/core/payment-link/services/payment-cron.service.ts b/src/subdomains/core/payment-link/services/payment-cron.service.ts index 598d4f90cb..54d78bb05c 100644 --- a/src/subdomains/core/payment-link/services/payment-cron.service.ts +++ b/src/subdomains/core/payment-link/services/payment-cron.service.ts @@ -1,76 +1,76 @@ -import { Injectable } from '@nestjs/common'; -import { CronExpression } from '@nestjs/schedule'; -import { Process } from 'src/shared/services/process.service'; -import { CronScope, DfxCron } from 'src/shared/utils/cron'; -import { CustomCronExpression } from 'src/shared/utils/custom-cron-expression'; -import { PaymentActivationService } from './payment-activation.service'; -import { PaymentBalanceService } from './payment-balance.service'; -import { PaymentLinkPaymentService } from './payment-link-payment.service'; -import { PaymentQuoteService } from './payment-quote.service'; - -@Injectable() -export class PaymentCronService { - constructor( - private readonly paymentLinkPaymentService: PaymentLinkPaymentService, - private readonly paymentActivationService: PaymentActivationService, - private readonly paymentQuoteService: PaymentQuoteService, - private readonly paymentBalanceService: PaymentBalanceService, - ) {} - - // The three jobs below split what used to be one decision. Writing and delivering have opposite - // requirements — a database write, a merchant webhook and a quote cancellation must happen once - // in the deployment, while the AsyncMap and the RxJS subject in PaymentLinkPaymentService are - // process-local and only reach a caller connected to the process that fires them. A single scope - // cannot satisfy both: `Worker` or `Api` leaves callers on every other process unreleased, - // `Both` repeats every write and every webhook. - // - // So the writing runs under the lease (`Worker`), and deliverPaymentUpdates delivers from the - // persisted state those writes leave behind, in every process, without a lease. It writes - // nothing and calls nothing outside its process, which is what allows it to run everywhere. - - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.PAYMENT_EXPIRATION }) - async processExpiredPayments(): Promise { - await this.paymentLinkPaymentService.processExpiredPayments(); - await this.paymentActivationService.processExpiredActivations(); - await this.paymentQuoteService.processExpiredQuotes(); - } - - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.PAYMENT_CONFIRMATIONS }) - async checkTxConfirmations(): Promise { - await this.paymentLinkPaymentService.checkTxConfirmations(); - } - - // Runs at 15 seconds rather than the minute the two jobs above run at, because it is the second - // hop of a chain: the writing job already costs up to a minute to notice, and this must not add - // another one to it. It stays cheap at that rate by looking only at what its own process holds — - // with no caller waiting and no device connected it issues no query at all. - // - // `useDelay: false` for the same reason: the jitter exists to spread jobs that do real work per - // run, and up to five seconds of it would be a third of this interval. - // - // Deliberately WITHOUT a `process` flag, and that is a correction. It carried one, and the flag - // looked like any other job's — but this job is not work, it is the bridge that carries a result - // from the process that wrote it to the process holding the connection. Switched off in the - // single-process setup nothing happens, because `doSave` delivers directly there; switched off - // after the split it silently cuts delivery to everything attached to the OTHER container: - // waiting callers of `GET /v1/paymentLink/payment/wait` and `GET /v1/lnurlp/wait/:id` hang until - // they give up, and connected devices are never told their payment went through. No alert sees - // it — every process still reports its role and a usable lease. - // - // A switch whose failure mode is invisible is worse than no switch. The same reasoning already - // applies to the role heartbeat and to `PaymentLinkGateway.checkConnections`: a mechanism the - // rest depends on does not get a kill switch. Whatever a switch here would have been used for — - // load, a misbehaving device — is reached by disabling the jobs that WRITE, which do have flags. - @DfxCron(CustomCronExpression.EVERY_15_SECONDS, { - scope: CronScope.BOTH, - useDelay: false, - }) - async deliverPaymentUpdates(): Promise { - await this.paymentLinkPaymentService.deliverPaymentUpdates(); - } - - @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.WORKER, process: Process.PAYMENT_FORWARDING }) - async forwardDeposits(): Promise { - await this.paymentBalanceService.forwardDeposits(); - } -} +import { Injectable } from '@nestjs/common'; +import { CronExpression } from '@nestjs/schedule'; +import { Process } from 'src/shared/services/process.service'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; +import { CustomCronExpression } from 'src/shared/utils/custom-cron-expression'; +import { PaymentActivationService } from './payment-activation.service'; +import { PaymentBalanceService } from './payment-balance.service'; +import { PaymentLinkPaymentService } from './payment-link-payment.service'; +import { PaymentQuoteService } from './payment-quote.service'; + +@Injectable() +export class PaymentCronService { + constructor( + private readonly paymentLinkPaymentService: PaymentLinkPaymentService, + private readonly paymentActivationService: PaymentActivationService, + private readonly paymentQuoteService: PaymentQuoteService, + private readonly paymentBalanceService: PaymentBalanceService, + ) {} + + // The three jobs below split what used to be one decision. Writing and delivering have opposite + // requirements — a database write, a merchant webhook and a quote cancellation must happen once + // in the deployment, while the AsyncMap and the device sink in PaymentLinkPaymentService are + // process-local and only reach a caller connected to the process that fires them. A single scope + // cannot satisfy both: `Worker` or `Api` leaves callers on every other process unreleased, + // `Both` repeats every write and every webhook. + // + // So the writing runs under the lease (`Worker`), and deliverPaymentUpdates delivers from the + // persisted state those writes leave behind, in every process, without a lease. It writes + // nothing and calls nothing outside its process, which is what allows it to run everywhere. + + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.PAYMENT_EXPIRATION }) + async processExpiredPayments(): Promise { + await this.paymentLinkPaymentService.processExpiredPayments(); + await this.paymentActivationService.processExpiredActivations(); + await this.paymentQuoteService.processExpiredQuotes(); + } + + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, process: Process.PAYMENT_CONFIRMATIONS }) + async checkTxConfirmations(): Promise { + await this.paymentLinkPaymentService.checkTxConfirmations(); + } + + // Runs at 15 seconds rather than the minute the two jobs above run at, because it is the second + // hop of a chain: the writing job already costs up to a minute to notice, and this must not add + // another one to it. It stays cheap at that rate by looking only at what its own process holds — + // with no caller waiting and no device connected it issues no query at all. + // + // `useDelay: false` for the same reason: the jitter exists to spread jobs that do real work per + // run, and up to five seconds of it would be a third of this interval. + // + // Deliberately WITHOUT a `process` flag, and that is a correction. It carried one, and the flag + // looked like any other job's — but this job is not work, it is the bridge that carries a result + // from the process that wrote it to the process holding the connection. Switched off in the + // single-process setup nothing happens, because `doSave` delivers directly there; switched off + // after the split it silently cuts delivery to everything attached to the OTHER container: + // waiting callers of `GET /v1/paymentLink/payment/wait` and `GET /v1/lnurlp/wait/:id` hang until + // they give up, and connected devices are never told their payment went through. No alert sees + // it — every process still reports its role and a usable lease. + // + // A switch whose failure mode is invisible is worse than no switch. The same reasoning already + // applies to the role heartbeat and to `PaymentLinkGateway.checkConnections`: a mechanism the + // rest depends on does not get a kill switch. Whatever a switch here would have been used for — + // load, a misbehaving device — is reached by disabling the jobs that WRITE, which do have flags. + @DfxCron(CustomCronExpression.EVERY_15_SECONDS, { + scope: CronScope.BOTH, + useDelay: false, + }) + async deliverPaymentUpdates(): Promise { + await this.paymentLinkPaymentService.deliverPaymentUpdates(); + } + + @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.WORKER, process: Process.PAYMENT_FORWARDING }) + async forwardDeposits(): Promise { + await this.paymentBalanceService.forwardDeposits(); + } +} diff --git a/src/subdomains/core/payment-link/services/payment-link-fee.service.ts b/src/subdomains/core/payment-link/services/payment-link-fee.service.ts index e6061f4aee..e7fa11c1aa 100644 --- a/src/subdomains/core/payment-link/services/payment-link-fee.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-fee.service.ts @@ -1,129 +1,130 @@ -import { Injectable, OnModuleInit } from '@nestjs/common'; -import { CronExpression } from '@nestjs/schedule'; -import { Environment, GetConfig } from 'src/config/config'; -import { MIN_FEE_RATE_SAT_VB } from 'src/integration/blockchain/bitcoin/services/bitcoin-based-fee.service'; -import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.enum'; -import { PaymentLinkBlockchains } from 'src/integration/blockchain/shared/util/blockchain.util'; -import { DfxLogger } from 'src/shared/services/dfx-logger'; -import { Process } from 'src/shared/services/process.service'; -import { CronScope, DfxCron } from 'src/shared/utils/cron'; -import { Util } from 'src/shared/utils/util'; -import { PayoutBitcoinService } from 'src/subdomains/supporting/payout/services/payout-bitcoin.service'; -import { PayoutFiroService } from 'src/subdomains/supporting/payout/services/payout-firo.service'; -import { BlockchainRegistryService } from '../../../../integration/blockchain/shared/services/blockchain-registry.service'; - -interface FeeCacheData { - timestamp: Date; - fee: number; -} - -@Injectable() -export class PaymentLinkFeeService implements OnModuleInit { - private readonly logger = new DfxLogger(PaymentLinkFeeService); - - private static readonly MINUTES_5 = 5 * 60; - - private readonly feeCache: Map; - - constructor( - private readonly blockchainRegistryService: BlockchainRegistryService, - private readonly payoutBitcoinService: PayoutBitcoinService, - private readonly payoutFiroService: PayoutFiroService, - ) { - this.feeCache = new Map(); - } - - onModuleInit() { - void this.updateFees(); - } - - // --- JOBS --- // - /** - * Scope Api, not Both: the cache it fills is a field of this service, and the only reader is - * getMinFee below. Following that chain out — createTransferAmount -> createTransferAmounts -> - * createQuote / createPayRequest, plus the Binance webhook handler — every caller is a request - * path or one of the Api-scoped payment crons. None of the Worker jobs in this domain reaches - * the cache: PaymentCronService::forwardDeposits takes its fee rate from BitcoinFeeService, and - * ::processExpiredPayments and ::checkTxConfirmations price nothing — they move a payment out of - * `Pending` and cancel or close what hangs off it. - * - * Both would also break the rule this scope mechanism introduced: a job that runs in every - * process must be harmless twice over, and this one queries gas prices for eight EVM chains - * plus Bitcoin and Firo fee estimates on every tick. In the worker those calls would spend - * quota to produce a value nothing there reads. - */ - @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.API, process: Process.UPDATE_BLOCKCHAIN_FEE }) - async updateFees(): Promise { - if (GetConfig().environment === Environment.LOC) return; - - for (const blockchain of PaymentLinkBlockchains) { - try { - const fee = await this.calculateFee(blockchain); - this.feeCache.set(blockchain, { - timestamp: new Date(), - fee, - }); - } catch (e) { - this.feeCache.delete(blockchain); - this.logger.error(`Failed to get fee for blockchain ${blockchain}:`, e); - } - } - } - - private async calculateFee(blockchain: Blockchain): Promise { - switch (blockchain) { - case Blockchain.BINANCE_PAY: - case Blockchain.KUCOIN_PAY: - case Blockchain.LIGHTNING: - case Blockchain.MONERO: - case Blockchain.ZANO: - case Blockchain.SOLANA: - case Blockchain.TRON: - case Blockchain.CARDANO: - case Blockchain.INTERNET_COMPUTER: - return 0; - - case Blockchain.ETHEREUM: - case Blockchain.SEPOLIA: - case Blockchain.ARBITRUM: - case Blockchain.OPTIMISM: - case Blockchain.BASE: - case Blockchain.GNOSIS: - case Blockchain.POLYGON: - case Blockchain.BINANCE_SMART_CHAIN: { - const client = this.blockchainRegistryService.getEvmClient(blockchain); - return +(await client.getRecommendedGasPrice()); - } - - // The customer minimum is the network's own minimum for an inbound payment to confirm — it - // must NOT include the CPFP/default margin from getSendFeeRate, which exists only for DFX's - // own outbound spends. The value differs per chain because the chains do, but neither carries - // the payout margin. - case Blockchain.BITCOIN: - // Bitcoin fees are user-adjustable and the chain can congest, so use the recommended - // (next-block) rate, which adapts to congestion — floored at the relay minimum so the - // advertised minimum is always relayable. - return Math.max(await this.payoutBitcoinService.getRecommendedFeeRate(), MIN_FEE_RATE_SAT_VB); - - case Blockchain.FIRO: - // Same principle as Bitcoin: Firo's own next-block rate without the payout margin, floored - // at the relay minimum so it stays relayable. The current OCP deposit address is transparent, - // so a Stack Wallet payment is a Spark-spend to it, whose fee sits at the relay floor and - // cannot be raised; Firo does not congest and its node usually returns no estimate, so this - // resolves to the relay floor in practice — exactly what that Spark-spend pays. A dedicated - // relay-floor cap belongs here only once a Spark `sm1…` deposit address is deployed, whose - // protocol-capped fee cannot follow a congestion-adaptive minimum. - return Math.max(await this.payoutFiroService.getRecommendedFeeRate(), MIN_FEE_RATE_SAT_VB); - } - } - - // --- PUBLIC METHODS --- // - async getMinFee(blockchain: Blockchain): Promise { - const cacheData = this.feeCache.get(blockchain); - if (!cacheData) return; - - if (Util.secondsDiff(cacheData.timestamp) > PaymentLinkFeeService.MINUTES_5) return; - - return cacheData.fee; - } -} +import { Injectable, OnModuleInit } from '@nestjs/common'; +import { CronExpression } from '@nestjs/schedule'; +import { Environment, GetConfig } from 'src/config/config'; +import { MIN_FEE_RATE_SAT_VB } from 'src/integration/blockchain/bitcoin/services/bitcoin-based-fee.service'; +import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.enum'; +import { PaymentLinkBlockchains } from 'src/integration/blockchain/shared/util/blockchain.util'; +import { DfxLogger } from 'src/shared/services/dfx-logger'; +import { Process } from 'src/shared/services/process.service'; +import { CronScope, DfxCron } from 'src/shared/utils/cron'; +import { Util } from 'src/shared/utils/util'; +import { PayoutBitcoinService } from 'src/subdomains/supporting/payout/services/payout-bitcoin.service'; +import { PayoutFiroService } from 'src/subdomains/supporting/payout/services/payout-firo.service'; +import { BlockchainRegistryService } from '../../../../integration/blockchain/shared/services/blockchain-registry.service'; + +interface FeeCacheData { + timestamp: Date; + fee: number; +} + +@Injectable() +export class PaymentLinkFeeService implements OnModuleInit { + private readonly logger = new DfxLogger(PaymentLinkFeeService); + + private static readonly MINUTES_5 = 5 * 60; + + private readonly feeCache: Map; + + constructor( + private readonly blockchainRegistryService: BlockchainRegistryService, + private readonly payoutBitcoinService: PayoutBitcoinService, + private readonly payoutFiroService: PayoutFiroService, + ) { + this.feeCache = new Map(); + } + + onModuleInit() { + void this.updateFees(); + } + + // --- JOBS --- // + /** + * Scope Api, not Both: the cache it fills is a field of this service, and the only reader is + * getMinFee below. Following that chain out — createTransferAmount -> createTransferAmounts -> + * createQuote / createPayRequest, plus the Binance webhook handler — every one of them is a + * request path. This job is the only `Api`-scoped cron in the domain, and it is the writer, not + * a reader. None of the Worker jobs in this domain reaches + * the cache: PaymentCronService::forwardDeposits takes its fee rate from BitcoinFeeService, and + * ::processExpiredPayments and ::checkTxConfirmations price nothing — they move a payment out of + * `Pending` and cancel or close what hangs off it. + * + * Both would also break the rule this scope mechanism introduced: a job that runs in every + * process must be harmless twice over, and this one queries gas prices for eight EVM chains + * plus Bitcoin and Firo fee estimates on every tick. In the worker those calls would spend + * quota to produce a value nothing there reads. + */ + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.API, process: Process.UPDATE_BLOCKCHAIN_FEE }) + async updateFees(): Promise { + if (GetConfig().environment === Environment.LOC) return; + + for (const blockchain of PaymentLinkBlockchains) { + try { + const fee = await this.calculateFee(blockchain); + this.feeCache.set(blockchain, { + timestamp: new Date(), + fee, + }); + } catch (e) { + this.feeCache.delete(blockchain); + this.logger.error(`Failed to get fee for blockchain ${blockchain}:`, e); + } + } + } + + private async calculateFee(blockchain: Blockchain): Promise { + switch (blockchain) { + case Blockchain.BINANCE_PAY: + case Blockchain.KUCOIN_PAY: + case Blockchain.LIGHTNING: + case Blockchain.MONERO: + case Blockchain.ZANO: + case Blockchain.SOLANA: + case Blockchain.TRON: + case Blockchain.CARDANO: + case Blockchain.INTERNET_COMPUTER: + return 0; + + case Blockchain.ETHEREUM: + case Blockchain.SEPOLIA: + case Blockchain.ARBITRUM: + case Blockchain.OPTIMISM: + case Blockchain.BASE: + case Blockchain.GNOSIS: + case Blockchain.POLYGON: + case Blockchain.BINANCE_SMART_CHAIN: { + const client = this.blockchainRegistryService.getEvmClient(blockchain); + return +(await client.getRecommendedGasPrice()); + } + + // The customer minimum is the network's own minimum for an inbound payment to confirm — it + // must NOT include the CPFP/default margin from getSendFeeRate, which exists only for DFX's + // own outbound spends. The value differs per chain because the chains do, but neither carries + // the payout margin. + case Blockchain.BITCOIN: + // Bitcoin fees are user-adjustable and the chain can congest, so use the recommended + // (next-block) rate, which adapts to congestion — floored at the relay minimum so the + // advertised minimum is always relayable. + return Math.max(await this.payoutBitcoinService.getRecommendedFeeRate(), MIN_FEE_RATE_SAT_VB); + + case Blockchain.FIRO: + // Same principle as Bitcoin: Firo's own next-block rate without the payout margin, floored + // at the relay minimum so it stays relayable. The current OCP deposit address is transparent, + // so a Stack Wallet payment is a Spark-spend to it, whose fee sits at the relay floor and + // cannot be raised; Firo does not congest and its node usually returns no estimate, so this + // resolves to the relay floor in practice — exactly what that Spark-spend pays. A dedicated + // relay-floor cap belongs here only once a Spark `sm1…` deposit address is deployed, whose + // protocol-capped fee cannot follow a congestion-adaptive minimum. + return Math.max(await this.payoutFiroService.getRecommendedFeeRate(), MIN_FEE_RATE_SAT_VB); + } + } + + // --- PUBLIC METHODS --- // + async getMinFee(blockchain: Blockchain): Promise { + const cacheData = this.feeCache.get(blockchain); + if (!cacheData) return; + + if (Util.secondsDiff(cacheData.timestamp) > PaymentLinkFeeService.MINUTES_5) return; + + return cacheData.fee; + } +} diff --git a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts index 24a28c7c67..bc399ae6d0 100644 --- a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts @@ -357,9 +357,10 @@ export class PaymentLinkPaymentService { * device connected they touch the database not at all. * * That it reads on a tick rather than subscribing is the choice this makes against CONTRIBUTING's - * "initial fetch + subscription for real-time data": the only subscription available here is the - * RxJS subject above, and that reaches no process but this one. A subscription that cannot see - * the writes it is meant to relay is not one. + * "initial fetch + subscription for real-time data": there is nothing here to subscribe TO. The + * writes happen in another process, and no in-process channel — a subject, an emitter, the sink + * above — can carry what it never sees. A subscription that cannot see the writes it is meant to + * relay is not one. */ async deliverPaymentUpdates(): Promise { await this.deliverToWaitingCallers(); From 908ead7fb617ebd907374a220c30f651471260b5 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:56 +0200 Subject: [PATCH 77/86] Do not let an unreachable lease table be worse than no lease at all MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cross-repo review found a blocker in the rollout, not in a single PR — the kind only that review can see. Between the production release of this application version (step 5) and the release that brings the lease alert (step 9), production runs with the lease in force and nothing watching it. `guardAcrossProcesses` does not look at the role, so under `CRON_ROLE=all` all 123 non-`BOTH` jobs go through the claim. An unreachable table — a missing grant, a migration that did not run, a database hiccup — made every one of them skip, payouts included, while every deployed rule stayed green. Under `all` the deployment runs ONE process. That is the shape the API had before this branch existed, with no lease at all. Skipping there made the lease strictly worse than its own absence, which cannot be the right answer for a mechanism that is supposed to add safety. So under `all` the job now runs, and the failure is logged loudly and carried out in the heartbeat. Under `api` or `worker` the skip stands: there the lease is the only separation, and running anyway is the double run it exists to prevent. Four more from the same round. The heartbeat is now written once at boot. It fires on fixed marks without jitter, so a process that comes up at :11 and misses :10 wrote nothing until :20 — against a twelve-minute window, an ordinary deploy could produce the critical "worker is silent" alarm. The gateway drops what it believes it told a device when a socket reports `'error'`. That is how `ws` reports a send on a socket that closed between the state check and the call, and the comment that dismissed it was wrong: it claimed the record would age out and the delivery retry. It cannot — the record ages on the same cutoff the query uses, so "record gone" and "payment still in the read" exclude each other. `PaymentLinkFeeService.getMinFee` loads on demand instead of answering `undefined`. CONTRIBUTING requires that of every cache read in a request path, and this PR made it bite: the refresh is now `Api`-scoped and leased, so among several API processes only one wins it per tick. And `SET LOCAL lock_timeout` is scoped to the whole migration batch, not to the statement below it — the comment said the first and argued the second. --- ...0000-AddPaymentLinkPaymentDeviceIdIndex.js | 20 +++++-- .../__tests__/cron-lease.service.spec.ts | 56 +++++++++++++++++++ .../__tests__/dfx-cron.service.spec.ts | 22 ++++++++ src/shared/services/cron-lease.service.ts | 32 ++++++++++- src/shared/services/dfx-cron.service.ts | 12 +++- .../__tests__/payment-link.gateway.spec.ts | 23 ++++++++ .../controllers/payment-link.gateway.ts | 22 ++++++-- .../services/payment-link-fee.service.ts | 32 ++++++++++- .../services/payment-link-payment.service.ts | 16 ++++++ 9 files changed, 218 insertions(+), 17 deletions(-) diff --git a/migration/1785620000000-AddPaymentLinkPaymentDeviceIdIndex.js b/migration/1785620000000-AddPaymentLinkPaymentDeviceIdIndex.js index 472fa96393..669dc79407 100644 --- a/migration/1785620000000-AddPaymentLinkPaymentDeviceIdIndex.js +++ b/migration/1785620000000-AddPaymentLinkPaymentDeviceIdIndex.js @@ -38,8 +38,14 @@ module.exports = class AddPaymentLinkPaymentDeviceIdIndex1785620000000 { * @param {QueryRunner} queryRunner */ async up(queryRunner) { - // SET LOCAL is scoped to the whole transaction. Bounds WAIT time to acquire the lock, not how - // long the lock is held. Set once: this migration has a single CREATE INDEX statement. + // SET LOCAL is scoped to the whole TRANSACTION, and under `migrationsTransactionMode: 'all'` + // that transaction is the entire pending batch — so this stays in force for every migration + // that runs after it in the same deployment, not only for the statement below. That is + // deliberate but worth knowing: a later migration that must wait on a lock inherits the five + // seconds and fails the whole release rather than waiting. Whoever adds one sets its own + // value. + // + // It bounds the WAIT for the lock, not how long the lock is held. await queryRunner.query(`SET LOCAL lock_timeout = '5s'`); await queryRunner.query(`CREATE INDEX "IDX_8a9b97a10b3db9c64d45ae4d38" ON "payment_link_payment" ("deviceId")`); } @@ -48,8 +54,14 @@ module.exports = class AddPaymentLinkPaymentDeviceIdIndex1785620000000 { * @param {QueryRunner} queryRunner */ async down(queryRunner) { - // SET LOCAL is scoped to the whole transaction. Bounds WAIT time to acquire the lock, not how - // long the lock is held. Set once: this migration has a single DROP INDEX statement. + // SET LOCAL is scoped to the whole TRANSACTION, and under `migrationsTransactionMode: 'all'` + // that transaction is the entire pending batch — so this stays in force for every migration + // that runs after it in the same deployment, not only for the statement below. That is + // deliberate but worth knowing: a later migration that must wait on a lock inherits the five + // seconds and fails the whole release rather than waiting. Whoever adds one sets its own + // value. + // + // It bounds the WAIT for the lock, not how long the lock is held. await queryRunner.query(`SET LOCAL lock_timeout = '5s'`); await queryRunner.query(`DROP INDEX "public"."IDX_8a9b97a10b3db9c64d45ae4d38"`); } diff --git a/src/shared/services/__tests__/cron-lease.service.spec.ts b/src/shared/services/__tests__/cron-lease.service.spec.ts index 5f236811d1..a35586f48e 100644 --- a/src/shared/services/__tests__/cron-lease.service.spec.ts +++ b/src/shared/services/__tests__/cron-lease.service.spec.ts @@ -55,6 +55,62 @@ describe('CronLeaseService', () => { expect(task).toHaveBeenCalledTimes(1); }); + describe('when the lease table cannot be reached', () => { + /** Every claim attempt fails, as it would with no table, no grant or no database. */ + const unreachable = () => + jest.fn().mockImplementation((sql: string) => { + if (sql.includes('INSERT INTO')) return Promise.reject(new Error('relation "cron_lease" does not exist')); + return Promise.resolve([]); + }); + + it('runs the task anyway under `all`, because one process is what that role means', async () => { + // The rollout puts this application version into production SEVERAL STEPS before the alert + // that reads the lease heartbeat. In that window an unreachable table would otherwise stop + // 123 of 139 jobs — payouts among them — with nothing to report it. + // + // Under `all` the deployment runs one process, which is the shape the API had before any + // lease existed. Skipping there would make the lease STRICTLY WORSE than its own absence. + process.env.CRON_ROLE = 'all'; + new ConfigService(GetConfig()); + + const { service } = buildService({ onQuery: unreachable() }); + const task = jest.fn().mockResolvedValue(undefined); + + await service.run('SomeService::job', task); + + expect(task).toHaveBeenCalledTimes(1); + }); + + it('does NOT run it under `worker`, where the lease is the only separation', async () => { + // The other direction, and the reason this is a role question rather than a blanket rule: + // with two processes, running anyway is the double run the lease exists to prevent. + process.env.CRON_ROLE = 'worker'; + new ConfigService(GetConfig()); + + const { service } = buildService({ onQuery: unreachable() }); + const task = jest.fn().mockResolvedValue(undefined); + + await service.run('SomeService::job', task); + + expect(task).not.toHaveBeenCalled(); + }); + + it('reports the failure in the heartbeat either way', async () => { + // Running anyway must not look healthy: the reason still has to reach the alert. + process.env.CRON_ROLE = 'all'; + new ConfigService(GetConfig()); + + const { service } = buildService({ onQuery: unreachable() }); + + await service.run('SomeService::job', jest.fn().mockResolvedValue(undefined)); + + const failures = service.takeFailures(); + + expect(failures.healthy).toBe(false); + expect(failures.count).toEqual(1); + }); + }); + it('does NOT run the task when another process holds the lease', async () => { // The claim statement returns no row when an unexpired lease belongs to someone else. This is // the case the whole mechanism exists for: the second process must stay out. diff --git a/src/shared/services/__tests__/dfx-cron.service.spec.ts b/src/shared/services/__tests__/dfx-cron.service.spec.ts index 96ce71f9df..0beb4db143 100644 --- a/src/shared/services/__tests__/dfx-cron.service.spec.ts +++ b/src/shared/services/__tests__/dfx-cron.service.spec.ts @@ -315,6 +315,28 @@ describe('DfxCronService', () => { // Everything it needs has to be IN the line and the line has to appear in both processes — // the three tests below pin exactly that, because none of it is visible at the call site. + it('writes one at boot, not only at the next ten-minute mark', () => { + // The job fires on fixed marks and does not jitter. A process that comes up at :11 and + // misses :10 would otherwise write nothing until :20 — against a twelve-minute alert window + // that turns an ordinary deploy into the critical "worker is silent" alarm. + const { service } = buildService([ + providerWithJob('someJob', { + expression: CronExpression.EVERY_MINUTE, + scope: CronScope.WORKER, + useDelay: false, + }), + ]); + + const info = jest.spyOn(service['logger'], 'info'); + + service.onModuleInit(); + + const lines = info.mock.calls.map(([line]) => line as string); + + expect(lines.some((line) => /CronRole \w+: registered \d+ of \d+ jobs/.test(line))).toBe(true); + expect(lines.some((line) => /CronRole \w+: heartbeat, \d+ jobs registered/.test(line))).toBe(true); + }); + it('runs in every process, so neither one is invisible to the alert', () => { // Were this scoped `worker`, the API process would stop reporting and the alert could no // longer distinguish "runs the wrong role" from "reports nothing". diff --git a/src/shared/services/cron-lease.service.ts b/src/shared/services/cron-lease.service.ts index f8f4193d2c..868f5a4871 100644 --- a/src/shared/services/cron-lease.service.ts +++ b/src/shared/services/cron-lease.service.ts @@ -1,6 +1,6 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { randomUUID } from 'crypto'; -import { Config } from 'src/config/config'; +import { Config, CronRole } from 'src/config/config'; import { DataSource } from 'typeorm'; import { DfxLogger } from './dfx-logger'; @@ -235,8 +235,34 @@ export class CronLeaseService implements OnModuleInit { acquired = await this.acquire(job); } catch (e) { this.recordFailure(e); - this.logger.error(`Skipping ${job}: could not reach the lease table`, e); - return; + + // Unreachable table, missing grant, database down. What happens next depends on the role, + // and the difference matters more than it looks. + // + // Under `all` the deployment runs ONE process — the same shape the API had before this + // branch existed, when no lease was involved at all. Skipping there would make an + // unreachable lease table STRICTLY WORSE than not having one: 123 of 139 jobs would stop, + // payouts included, and between the rollout of this application version and the rollout of + // the alert that reads the heartbeat there is no rule that would say so. So the job runs. + // Two processes on `all` would then run it twice — exactly as they would have before, and + // the `role-mismatch` rule reports that pair once it exists. + // + // Under `api` or `worker` the lease is the only thing keeping the job to one process, and + // its absence is not recoverable by running anyway. There the skip stands, and the + // heartbeat carries the reason out. + if (Config.cronRole !== CronRole.ALL) { + this.logger.error(`Skipping ${job}: could not reach the lease table`, e); + return; + } + + this.logger.error( + `Running ${job} WITHOUT a lease: could not reach the lease table, and CRON_ROLE=all runs ` + + `one process — not running it would be worse than the single-process setup this ` + + `replaces`, + e, + ); + + return task(); } if (!acquired) { diff --git a/src/shared/services/dfx-cron.service.ts b/src/shared/services/dfx-cron.service.ts index 3fcf8aa202..867cd5d78d 100644 --- a/src/shared/services/dfx-cron.service.ts +++ b/src/shared/services/dfx-cron.service.ts @@ -6,9 +6,9 @@ import { Config, CronRole } from 'src/config/config'; import { DisabledProcess } from 'src/shared/services/process.service'; import { CronScope, DFX_CRONJOB_PARAMS, DfxCron, DfxCronExpression, DfxCronParams } from 'src/shared/utils/cron'; import { LockClass } from 'src/shared/utils/lock'; -import { CronLeaseService } from './cron-lease.service'; import { Util } from 'src/shared/utils/util'; import { CustomCronExpression } from '../utils/custom-cron-expression'; +import { CronLeaseService } from './cron-lease.service'; import { DfxLogger } from './dfx-logger'; interface CronJobData { @@ -76,6 +76,16 @@ export class DfxCronService implements OnModuleInit { this.registeredCount = registered.length; this.logger.info(`CronRole ${Config.cronRole}: registered ${registered.length} of ${total} jobs (${byScope})`); + + // And a heartbeat right away, not only at the next ten-minute mark. The line above is a + // DIFFERENT one — the alerts read the heartbeat, and a process that has just started would + // otherwise be missing from it for up to ten minutes. + // + // That is not a theoretical gap: the job fires on fixed marks (`useDelay: false`), so a + // process that comes up at :11 and misses :10 writes nothing until :20. Against a + // twelve-minute window, an ordinary deploy could then produce the "worker is silent" alarm — + // the critical one this whole split rests on. + this.reportRole(); } /** diff --git a/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts b/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts index 419a240d8f..0f63b831f0 100644 --- a/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts +++ b/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts @@ -58,6 +58,7 @@ describe('PaymentLinkGateway', () => { paymentService = { useDeviceSink: jest.fn().mockImplementation((sink) => (deliver = sink)), useDeviceSource: jest.fn(), + forgetDeliveries: jest.fn(), } as unknown as jest.Mocked; gateway = new PaymentLinkGateway(paymentService); @@ -221,6 +222,28 @@ describe('PaymentLinkGateway', () => { expect(deliver({ id: 'pos-unknown', command: 'show-paid' })).toBe(false); }); + it('forgets what a device was told when its socket reports an error', () => { + // The one failure the state check cannot see: `ws` reports a send on a socket that closed + // mid-call asynchronously, through `'error'` — the sink has already answered `true` and the + // delivery has recorded the command. Letting the record age out does not repair that: it ages + // on the same cutoff the query uses, so it never outlives the payment's place in the read. + const client = connect('pos-1'); + + client.fire('error'); + + expect(paymentService.forgetDeliveries).toHaveBeenCalledWith('pos-1'); + }); + + it('keeps what a device was told when its socket closes in order', () => { + // The other direction, and the reason the record exists: an orderly close says nothing about + // a command that did go out. Forgetting here would repeat every command on every reconnect. + const client = connect('pos-1'); + + client.fire('close'); + + expect(paymentService.forgetDeliveries).not.toHaveBeenCalled(); + }); + it('reports nothing delivered when the socket is closing, and drops it', () => { // The path `send` cannot report: on a CLOSING or CLOSED socket `ws` takes the call quietly // and raises the failure through `'error'` later. Without reading the state first the sink diff --git a/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts b/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts index 77ecc7400c..93e483e576 100644 --- a/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts +++ b/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts @@ -117,7 +117,14 @@ export class PaymentLinkGateway implements OnGatewayConnection, OnModuleInit { // Bound to the socket, and to every way it can end: an aborted connection reports `error`, and // binding to `close` alone left it registered. Removal is idempotent, so both firing is fine. client.on('close', () => this.removeClient(device, clientId)); - client.on('error', () => this.removeClient(device, clientId)); + // `error` does one thing more than `close`: it is how `ws` reports a send on a socket that + // closed between the delivery's state check and the call itself. The delivery has already + // recorded that command as sent, so the record has to go — see + // PaymentLinkPaymentService.forgetDeliveries for why `close` must NOT do the same. + client.on('error', () => { + this.removeClient(device, clientId); + this.paymentService.forgetDeliveries(device); + }); client.on('pong', () => this.markResponsive(device, clientId)); } @@ -159,11 +166,14 @@ export class PaymentLinkGateway implements OnGatewayConnection, OnModuleInit { * One failing socket does not stop the others: a device with two connections is still reachable * through the second, and that counts as delivered. * - * What remains uncovered: a socket that closes BETWEEN the check and the send. That window is - * the width of one synchronous call, and the delivery survives it the same way it survives a - * process restart — the payment stays inside the read until its own end, so a later tick tries - * again as soon as the device reconnects and the record has aged out. What it does not survive - * is being told `true` for a socket that was already gone, which is what this closes. + * What the state check cannot cover: a socket that closes BETWEEN the check and the send. `ws` + * reports that one asynchronously through `'error'`, so this has already answered `true`. + * + * Waiting for the delivery's record to age out does NOT repair it, and an earlier version of + * this comment claimed it did. That record ages on the same cutoff the delivery's query uses, + * so "the record is gone" and "the payment is still in the read" exclude each other by + * construction — the retry it promised could never happen. The repair is in the `'error'` + * handler above, which drops the record so the next tick sends the state again. */ private sendMessage(device: PaymentDevice): boolean { const connections = this.clients.get(device.id); diff --git a/src/subdomains/core/payment-link/services/payment-link-fee.service.ts b/src/subdomains/core/payment-link/services/payment-link-fee.service.ts index e7fa11c1aa..2d654511ac 100644 --- a/src/subdomains/core/payment-link/services/payment-link-fee.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-fee.service.ts @@ -119,12 +119,38 @@ export class PaymentLinkFeeService implements OnModuleInit { } // --- PUBLIC METHODS --- // + /** + * Loads on demand when the cache has nothing usable, rather than answering `undefined`. + * + * CONTRIBUTING: "A cache read in a request path must load on demand … A cron job may refresh + * it, but must not be the only thing filling it." The job above refreshes; this is the load. + * + * Before the split that distinction did not bite: one process ran the refresh and served the + * requests, so a filled cache and a reading request were the same process. Now the refresh is + * `Api`-scoped AND leased, so among several API processes only one wins it per tick — the + * others would serve `undefined` for a fee they could have fetched, and `createQuote` would + * price without it. + * + * Only the blockchain that was asked for is loaded, not the whole set: a request pays for what + * it needs, and the job stays the thing that keeps the rest warm. + */ async getMinFee(blockchain: Blockchain): Promise { const cacheData = this.feeCache.get(blockchain); - if (!cacheData) return; + const usable = cacheData && Util.secondsDiff(cacheData.timestamp) <= PaymentLinkFeeService.MINUTES_5; + if (usable) return cacheData.fee; + + try { + const fee = await this.calculateFee(blockchain); + this.feeCache.set(blockchain, { timestamp: new Date(), fee }); - if (Util.secondsDiff(cacheData.timestamp) > PaymentLinkFeeService.MINUTES_5) return; + return fee; + } catch (e) { + // Same shape as the job's own failure handling: a fee source that cannot be reached leaves + // the caller without a minimum, which is what it would have had anyway. Logged rather than + // thrown, so one unreachable chain does not fail a quote for the others. + this.logger.error(`Failed to load fee for blockchain ${blockchain} on demand:`, e); - return cacheData.fee; + return undefined; + } } } diff --git a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts index bc399ae6d0..ab56b4bc75 100644 --- a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts @@ -430,6 +430,22 @@ export class PaymentLinkPaymentService { } } + /** + * Drops what this process believes it told a device, so the next tick tells it again. + * + * Called by the gateway when a socket reports `'error'` — the one signal that a command may + * have failed AFTER the sink answered `true`. `ws` raises a send on a socket that closed + * mid-call that way, and nothing else can see it: the state was open when it was read, and the + * call did not throw. + * + * Deliberately not called on `'close'`. An orderly close is not evidence that anything failed, + * and forgetting there would repeat every command on every reconnect — which is what the record + * exists to prevent. + */ + forgetDeliveries(deviceId: string): void { + this.deviceDeliveries.delete(deviceId); + } + /** What has been delivered to a device so far, empty for one nothing has been sent to yet. */ private deliveriesFor(deviceId: string): DeviceDeliveries { const delivered = this.deviceDeliveries.get(deviceId) ?? new Map(); From 2fb60aa5fbd94f7189b7db48504c28abb9699ea8 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:58 +0200 Subject: [PATCH 78/86] Give the lease-less run the same guarantees as every other, and say so MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round twelve found that the previous round's fix left two holes and eleven sentences describing the behaviour it replaced. The fail-open path called the task directly. That skipped the two things the healthy path does between the claim and the call, and neither is about the lease: the run never entered `inFlight`, so `shutdown` gave it no grace period and reported nothing was running; and it started even if shutdown had begun during the failed claim — a WIDER window than the healthy one, because a failing attempt runs to the database timeout. A payout could be cut off part-way with the shutdown log saying the process was idle. Both paths now go through `track`, which owns exactly those two: the late shutdown check and the `inFlight` entry. What the claim-holding path adds on top — stop renewing, hand the claim back — it passes in. Eleven statements still described the lease as fail-closed everywhere. Five in this repo: the boot error line a reader sees in production, the `run` doc twelve lines above the code doing the opposite, `takeFailures`, `reportRole` and `onModuleInit`. They now say what depends on the role and what does not — under `api`/`worker` the jobs stop, under `all` they run without a claim, and both are states to fix, which is why the heartbeat carries either out. The boot line branches too: under `all` it would have told an operator the opposite of what the process does. --- .../__tests__/cron-lease.service.spec.ts | 37 ++++++++ src/shared/services/cron-lease.service.ts | 95 +++++++++++++------ src/shared/services/dfx-cron.service.ts | 9 +- 3 files changed, 110 insertions(+), 31 deletions(-) diff --git a/src/shared/services/__tests__/cron-lease.service.spec.ts b/src/shared/services/__tests__/cron-lease.service.spec.ts index a35586f48e..7d9b920234 100644 --- a/src/shared/services/__tests__/cron-lease.service.spec.ts +++ b/src/shared/services/__tests__/cron-lease.service.spec.ts @@ -95,6 +95,43 @@ describe('CronLeaseService', () => { expect(task).not.toHaveBeenCalled(); }); + it('waits for a lease-less run at shutdown, like any other', async () => { + // The fail-open path once called the task directly and skipped the `inFlight` entry with + // it. A payout started that way was invisible to `shutdown`: no grace period, and the + // "still running" warning said nothing was. + process.env.CRON_ROLE = 'all'; + new ConfigService(GetConfig()); + + const { service } = buildService({ onQuery: unreachable() }); + let finish: () => void; + const task = jest.fn().mockImplementation(() => new Promise((resolve) => (finish = resolve))); + + const run = service.run('SomeService::job', task); + await settle(); + + expect(service['inFlight'].has('SomeService::job')).toBe(true); + + finish(); + await run; + + expect(service['inFlight'].has('SomeService::job')).toBe(false); + }); + + it('does not start a lease-less run once shutdown has begun', async () => { + // The claim attempt runs to the database timeout, so this window is WIDER than the healthy + // one — and a run started inside it would be cut off part-way through. + process.env.CRON_ROLE = 'all'; + new ConfigService(GetConfig()); + + const { service } = buildService({ onQuery: unreachable() }); + const task = jest.fn().mockResolvedValue(undefined); + + service['shuttingDown'] = true; + await service.run('SomeService::job', task); + + expect(task).not.toHaveBeenCalled(); + }); + it('reports the failure in the heartbeat either way', async () => { // Running anyway must not look healthy: the reason still has to reach the alert. process.env.CRON_ROLE = 'all'; diff --git a/src/shared/services/cron-lease.service.ts b/src/shared/services/cron-lease.service.ts index 868f5a4871..b9295858bd 100644 --- a/src/shared/services/cron-lease.service.ts +++ b/src/shared/services/cron-lease.service.ts @@ -132,11 +132,15 @@ export class CronLeaseService implements OnModuleInit { * Reads the lease table once, so a process that cannot use it says so at start-up. * * Without the table — a process started before the migration ran, a revoked grant, a database - * that is not there yet — every worker- and api-scoped job fails its claim and is skipped. That - * is the correct behaviour, and CONTRIBUTING asks for exactly it where the alternative is - * proceeding on an unverified assumption. The problem it left behind is a reporting one: the - * skip is indistinguishable from a job that had nothing to do, and the role heartbeat is scoped - * `both`, so it is exempt from the lease and keeps reporting a healthy process. + * that is not there yet — every worker- and api-scoped job fails its claim. Under `api` or + * `worker` it is then skipped, which is the correct behaviour and what CONTRIBUTING asks for + * where the alternative is proceeding on an unverified assumption. Under `all` it runs anyway, + * because that role is one process and stopping would be worse than the setup this replaces. + * + * Both outcomes have the same reporting problem, which is why this line exists: a skip is + * indistinguishable from a job that had nothing to do, a claim-less run from a normal one, and + * the role heartbeat is scoped `both` — exempt from the lease, and reporting a healthy process + * in either case. * * This does not take the boot down. A crash loop here would be loud, but it would also be * self-inflicted during the very rollout that introduces the table: the migration ships with the @@ -152,7 +156,11 @@ export class CronLeaseService implements OnModuleInit { } catch (e) { this.recordFailure(e); this.logger.error( - 'The cron lease table cannot be read: every worker- and api-scoped job will be skipped on every tick', + Config.cronRole === CronRole.ALL + ? 'The cron lease table cannot be read. CRON_ROLE=all runs one process, so worker- and ' + + 'api-scoped jobs keep running WITHOUT a claim rather than stopping — the same shape ' + + 'this deployment had before the lease existed. Fix the table before splitting the roles.' + : 'The cron lease table cannot be read: every worker- and api-scoped job will be skipped ' + 'on every tick', e, ); } @@ -213,9 +221,14 @@ export class CronLeaseService implements OnModuleInit { /** * Runs `task` only if this process can claim the lease, and keeps the claim alive meanwhile. * - * Failing to reach the database means NOT running: a job that moves money must not proceed on + * Failing to reach the database means NOT running — under `api` or `worker`. There the lease is + * the only thing keeping the job to one process, and a job that moves money must not proceed on * the assumption that it is probably alone. The caller sees the same outcome as a job whose - * lease is held elsewhere — it simply does not run this cycle and tries again on the next. + * lease is held elsewhere: it does not run this cycle and tries again on the next. + * + * Under `all` it means running anyway. That role IS one process, the shape this deployment had + * before any lease existed, so stopping there would make an unreachable table strictly worse + * than its own absence — see the branch in the catch below. * * `reportContention` marks jobs for which losing the race is not a normal outcome. For a worker * job it is: the other worker holds the lease and is doing the work, and the result lands in the @@ -262,7 +275,13 @@ export class CronLeaseService implements OnModuleInit { e, ); - return task(); + // Through `track` rather than as a bare `task()`. The two things the healthy path does + // between here and the call are not about the lease at all, and skipping them would make + // this run less safe than every other: it would be invisible to `shutdown` — no grace, and + // absent from the "still running" warning — and it would start even if shutdown had begun + // during the failed claim, which is a longer window than the healthy one because the + // attempt runs to the database timeout. + return this.track(job, task); } if (!acquired) { @@ -275,30 +294,51 @@ export class CronLeaseService implements OnModuleInit { return; } - // Claiming the lease is a round trip, and shutdown can begin during it. Hand the claim straight - // back rather than start under it, so the successor does not sit out the expiry for a job that - // never ran. + // Claiming the lease is a round trip, and shutdown can begin during it. `track` below checks + // for that as its first act and hands the claim straight back rather than starting under it, + // so the successor does not sit out the expiry for a job that never ran. // - // This is best effort, not a guarantee: a job still inside its claim is not in `inFlight` yet, - // so `shutdown` does not wait for it, and the process can exit before the release below runs. + // This is best effort, not a guarantee: a job still inside its claim is not in `inFlight` + // yet, so `shutdown` does not wait for it, and the process can exit before the release runs. // What then remains is a claim nobody holds — it lapses within the TTL like any other, which // is the bound that always applies. The release only ever shortens that wait. + const renewal = this.keepAlive(job); + + return this.track(job, task, () => { + renewal.stop(); + + return this.release(job).catch((e) => { + this.recordFailure(e); + this.logger.error(`Could not release the lease for ${job}`, e); + }); + }); + } + + /** + * Runs a task as one this process is known to be running. + * + * Everything a run needs regardless of whether it holds a claim: the shutdown check that must + * happen as late as possible, and the `inFlight` entry that makes the run visible to + * `shutdown`. Both were once written out only on the path that holds a lease, which left the + * lease-less path — the one taken when the table cannot be reached under `all` — without either. + * + * `after` is what the claim-holding path adds: stop renewing, hand the claim back. The + * lease-less path has nothing to hand back. + */ + private async track(job: string, task: () => Promise, after?: () => Promise): Promise { + // As late as possible, because the step before it is a round trip: shutdown can begin while a + // claim is being taken, or while a failing attempt runs to its timeout. A run started after + // that point is not in the set `shutdown` waits on and would be cut off part-way through. if (this.shuttingDown) { - await this.release(job).catch(() => undefined); + await after?.(); return; } - const renewal = this.keepAlive(job); - const run = (async () => { try { await task(); } finally { - renewal.stop(); - await this.release(job).catch((e) => { - this.recordFailure(e); - this.logger.error(`Could not release the lease for ${job}`, e); - }); + await after?.(); this.inFlight.delete(job); } })(); @@ -361,11 +401,12 @@ export class CronLeaseService implements OnModuleInit { /** * The state of the lease layer, for the role heartbeat to report. * - * Read rather than pushed: a lease that cannot reach its table stops every worker- and - * api-scoped job, and no other line says so — the jobs simply do not run. `healthy` stays false - * until an operation succeeds, so a role whose jobs are all sitting out keeps reporting it - * instead of falling quiet after the first window. The counter is per window; the last message - * is not, so an unhealthy report always names something. + * Read rather than pushed, and it has to be read under BOTH roles because it means different + * things: under `api` or `worker` a lease that cannot reach its table stops every worker- and + * api-scoped job, and no other line says so; under `all` it stops nothing, but the jobs then run + * without a claim, which is the state to fix before the roles are split. `healthy` stays false + * until an operation succeeds, so neither case falls quiet after the first window. The counter + * is per window; the last message is not, so an unhealthy report always names something. */ takeFailures(): { healthy: boolean; count: number; last?: string } { const taken = { healthy: this.healthy, count: this.failures, last: this.lastFailure }; diff --git a/src/shared/services/dfx-cron.service.ts b/src/shared/services/dfx-cron.service.ts index 867cd5d78d..a7f3743a7e 100644 --- a/src/shared/services/dfx-cron.service.ts +++ b/src/shared/services/dfx-cron.service.ts @@ -131,10 +131,11 @@ export class DfxCronService implements OnModuleInit { const line = `CronRole ${Config.cronRole}: heartbeat, ${this.registeredCount} jobs registered`; const lease = this.leases.takeFailures(); - // A job that cannot take its lease does not run, and nothing else says so — the skip looks - // exactly like a job with nothing to do. This job is scope `both` and therefore exempt from - // the lease itself, so it keeps reporting while everything it counts is sitting out: a count - // of REGISTERED jobs cannot see that. + // What the lease layer costs when it is broken depends on the role, and neither case says so + // by itself: under `api`/`worker` the jobs are skipped, and a skip looks exactly like a job + // with nothing to do; under `all` they run without a claim, which looks like nothing at all. + // This job is scope `both` and therefore exempt from the lease itself, so it keeps reporting + // in both cases — a count of REGISTERED jobs sees neither. if (lease.healthy) return this.logger.info(`${line}, lease ok`); this.logger.error( From f4474d711db4a54be430a36c005c411abc195946 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:31:59 +0200 Subject: [PATCH 79/86] Keep the fee warm-up out of the worker, and make the on-demand load safe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three things the on-demand load in getMinFee left open, all of them consequences of the same move — a value that used to be produced by one process for itself is now produced in one process and read in another. onModuleInit ran the full sweep in EVERY process. It is a Nest lifecycle hook, not a cron job, so the `Api` scope on updateFees never reached it: at boot the worker queried gas prices for eight EVM chains plus Bitcoin and Firo estimates to fill a map nothing in that process reads — exactly what the scope exists to prevent. The hook now asks whether this process serves requests, which is the property that makes a warm cache worth anything; the role-to-scope table stays in one place, in runsInThisRole. The load carried no LOC guard, while the job it complements does. Locally there are no node connections to ask, so the cache stays empty by design and every request would have run into the timeout of every one of those calls. It answers undefined again, as it did before this method loaded anything. And it started one fetch per caller: the cache is written when a load RESOLVES, so a burst of quotes for the same chain all saw the same empty entry. That is not hypothetical — updateFees is leased, so among several API processes only one wins it per tick and the others rely on this path for a whole minute. Concurrent callers now share the load in flight; a failed one is dropped in `finally`, so the next caller retries instead of inheriting it. Proven by 12 tests, each of the three guards checked against its own removal. --- .../payment-link-fee.service.spec.ts | 106 ++++++++++++++++++ .../services/payment-link-fee.service.ts | 69 +++++++++++- 2 files changed, 170 insertions(+), 5 deletions(-) diff --git a/src/subdomains/core/payment-link/services/__tests__/payment-link-fee.service.spec.ts b/src/subdomains/core/payment-link/services/__tests__/payment-link-fee.service.spec.ts index 9f2634deb8..5d2ae2ccb3 100644 --- a/src/subdomains/core/payment-link/services/__tests__/payment-link-fee.service.spec.ts +++ b/src/subdomains/core/payment-link/services/__tests__/payment-link-fee.service.spec.ts @@ -1,9 +1,29 @@ +import { ConfigService, CronRole, Environment, GetConfig } from 'src/config/config'; import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.enum'; import { BlockchainRegistryService } from 'src/integration/blockchain/shared/services/blockchain-registry.service'; import { PayoutBitcoinService } from 'src/subdomains/supporting/payout/services/payout-bitcoin.service'; import { PayoutFiroService } from 'src/subdomains/supporting/payout/services/payout-firo.service'; import { PaymentLinkFeeService } from '../payment-link-fee.service'; +/** Rebuilds the global `Config` from the current environment, the way the config spec does. */ +function withEnv(vars: Record): () => void { + const previous = Object.keys(vars).map((key) => [key, process.env[key]] as const); + + for (const [key, value] of Object.entries(vars)) { + if (value == null) delete process.env[key]; + else process.env[key] = value; + } + new ConfigService(GetConfig()); + + return () => { + for (const [key, value] of previous) { + if (value == null) delete process.env[key]; + else process.env[key] = value; + } + new ConfigService(GetConfig()); + }; +} + describe('PaymentLinkFeeService', () => { let service: PaymentLinkFeeService; let blockchainRegistryService: jest.Mocked; @@ -63,4 +83,90 @@ describe('PaymentLinkFeeService', () => { expect(fee).toBe(1); }); }); + + // --- getMinFee() Tests --- // + + describe('getMinFee()', () => { + let restore: () => void; + + afterEach(() => restore?.()); + + it('should load on demand when the cache is empty', async () => { + restore = withEnv({ ENVIRONMENT: Environment.DEV }); + + await expect(service.getMinFee(Blockchain.BITCOIN)).resolves.toBe(4); + expect(payoutBitcoinService.getRecommendedFeeRate).toHaveBeenCalledTimes(1); + }); + + it('should serve the second call from the cache the first one filled', async () => { + restore = withEnv({ ENVIRONMENT: Environment.DEV }); + + await service.getMinFee(Blockchain.BITCOIN); + await expect(service.getMinFee(Blockchain.BITCOIN)).resolves.toBe(4); + + expect(payoutBitcoinService.getRecommendedFeeRate).toHaveBeenCalledTimes(1); + }); + + it('should start ONE load for concurrent callers of the same blockchain', async () => { + restore = withEnv({ ENVIRONMENT: Environment.DEV }); + + // The cache is written when the load resolves, so without the in-flight map each of these + // would see the same empty entry and start its own call. + const fees = await Promise.all([ + service.getMinFee(Blockchain.BITCOIN), + service.getMinFee(Blockchain.BITCOIN), + service.getMinFee(Blockchain.BITCOIN), + ]); + + expect(fees).toEqual([4, 4, 4]); + expect(payoutBitcoinService.getRecommendedFeeRate).toHaveBeenCalledTimes(1); + }); + + it('should let the next caller retry after a failed load instead of inheriting it', async () => { + restore = withEnv({ ENVIRONMENT: Environment.DEV }); + payoutBitcoinService.getRecommendedFeeRate.mockRejectedValueOnce(new Error('node down')); + + await expect(service.getMinFee(Blockchain.BITCOIN)).resolves.toBeUndefined(); + await expect(service.getMinFee(Blockchain.BITCOIN)).resolves.toBe(4); + + expect(payoutBitcoinService.getRecommendedFeeRate).toHaveBeenCalledTimes(2); + }); + + it('should not reach out to any fee source on LOC', async () => { + // There are no node connections locally, so the job returns early and never fills the cache. + // Loading on demand here would run into a timeout on every request. + restore = withEnv({ ENVIRONMENT: Environment.LOC }); + + await expect(service.getMinFee(Blockchain.BITCOIN)).resolves.toBeUndefined(); + expect(payoutBitcoinService.getRecommendedFeeRate).not.toHaveBeenCalled(); + }); + }); + + // --- onModuleInit() Tests --- // + + describe('onModuleInit()', () => { + let restore: () => void; + + afterEach(() => restore?.()); + + it('should NOT warm the cache in the worker process', () => { + // The hook runs in every process regardless of CRON_ROLE; the scope on `updateFees` does not + // reach it. Unguarded it would query every fee source to fill a map nothing there reads. + restore = withEnv({ CRON_ROLE: CronRole.WORKER }); + const updateFees = jest.spyOn(service, 'updateFees').mockResolvedValue(); + + service.onModuleInit(); + + expect(updateFees).not.toHaveBeenCalled(); + }); + + it.each([CronRole.ALL, CronRole.API])('should warm the cache where requests land (%s)', (role) => { + restore = withEnv({ CRON_ROLE: role }); + const updateFees = jest.spyOn(service, 'updateFees').mockResolvedValue(); + + service.onModuleInit(); + + expect(updateFees).toHaveBeenCalledTimes(1); + }); + }); }); diff --git a/src/subdomains/core/payment-link/services/payment-link-fee.service.ts b/src/subdomains/core/payment-link/services/payment-link-fee.service.ts index 2d654511ac..309d0f757b 100644 --- a/src/subdomains/core/payment-link/services/payment-link-fee.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-fee.service.ts @@ -1,6 +1,6 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; -import { Environment, GetConfig } from 'src/config/config'; +import { Config, CronRole, Environment, GetConfig } from 'src/config/config'; import { MIN_FEE_RATE_SAT_VB } from 'src/integration/blockchain/bitcoin/services/bitcoin-based-fee.service'; import { Blockchain } from 'src/integration/blockchain/shared/enums/blockchain.enum'; import { PaymentLinkBlockchains } from 'src/integration/blockchain/shared/util/blockchain.util'; @@ -25,6 +25,17 @@ export class PaymentLinkFeeService implements OnModuleInit { private readonly feeCache: Map; + /** + * The loads currently in flight, one entry per blockchain, so concurrent readers of a cold cache + * share one fetch instead of each starting their own. Without it a burst of quotes for the same + * chain would fire one gas-price call per request: the cache is only written when a load + * RESOLVES, so every request arriving until then sees the same empty entry. + * + * That burst is not hypothetical here. `updateFees` is leased, so among several API processes + * only one wins it per tick and the others rely on this path for a whole minute. + */ + private readonly loading = new Map>(); + constructor( private readonly blockchainRegistryService: BlockchainRegistryService, private readonly payoutBitcoinService: PayoutBitcoinService, @@ -33,7 +44,24 @@ export class PaymentLinkFeeService implements OnModuleInit { this.feeCache = new Map(); } + /** + * Warms the cache at boot — but only where something reads it. + * + * `onModuleInit` runs in EVERY process, whatever `CRON_ROLE` says: it is a Nest lifecycle hook, + * not a cron job, so the scope on `updateFees` below does not reach it. Called unconditionally + * it would undo that scope at boot and make the worker do the one thing the scope exists to + * prevent — query gas prices for eight EVM chains plus Bitcoin and Firo to fill a map nothing + * in that process ever reads. + * + * The condition is deliberately about serving requests rather than about which jobs this role + * registers. This cache has exactly one reader, `getMinFee`, and every path to it is a request + * path; a warm cache is worth something where those requests land and nowhere else. Stating it + * that way also keeps the scope table in `runsInThisRole` the single place that maps roles to + * scopes, instead of copying it here where a later change would not reach it. + */ onModuleInit() { + if (Config.cronRole === CronRole.WORKER) return; + void this.updateFees(); } @@ -139,11 +167,14 @@ export class PaymentLinkFeeService implements OnModuleInit { const usable = cacheData && Util.secondsDiff(cacheData.timestamp) <= PaymentLinkFeeService.MINUTES_5; if (usable) return cacheData.fee; - try { - const fee = await this.calculateFee(blockchain); - this.feeCache.set(blockchain, { timestamp: new Date(), fee }); + // The same guard the job carries, for the same reason. On LOC there are no node connections to + // ask, so `updateFees` never fills the cache and this path would run into the timeout of every + // one of those calls on every request. Answering `undefined` is what a local environment + // returned before this method loaded anything, and the callers already handle it. + if (GetConfig().environment === Environment.LOC) return undefined; - return fee; + try { + return await this.loadFee(blockchain); } catch (e) { // Same shape as the job's own failure handling: a fee source that cannot be reached leaves // the caller without a minimum, which is what it would have had anyway. Logged rather than @@ -153,4 +184,32 @@ export class PaymentLinkFeeService implements OnModuleInit { return undefined; } } + + // --- HELPER METHODS --- // + /** + * One load per blockchain at a time; concurrent callers await the one already running. + * + * The entry is removed in `finally`, before the promise is handed out. A failed load therefore + * leaves nothing behind that a later call would await forever, and the next caller retries + * rather than inheriting the failure — the cache is a fee that expires, not a decision. + */ + private loadFee(blockchain: Blockchain): Promise { + const running = this.loading.get(blockchain); + if (running) return running; + + const load = (async () => { + try { + const fee = await this.calculateFee(blockchain); + this.feeCache.set(blockchain, { timestamp: new Date(), fee }); + + return fee; + } finally { + this.loading.delete(blockchain); + } + })(); + + this.loading.set(blockchain, load); + + return load; + } } From 3578f6581f4a69f04f735315c68e5c20485aacf9 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:32:01 +0200 Subject: [PATCH 80/86] State what holds instead of what an earlier comment said MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The comment explained why waiting for the record to age out does not repair a socket that closed between the state check and the send — and framed it as a correction of a previous version of itself. The reason is the same either way, and the reader of a merged file has no previous version to compare against. --- .../payment-link/controllers/payment-link.gateway.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts b/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts index 93e483e576..9d530c3823 100644 --- a/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts +++ b/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts @@ -169,11 +169,11 @@ export class PaymentLinkGateway implements OnGatewayConnection, OnModuleInit { * What the state check cannot cover: a socket that closes BETWEEN the check and the send. `ws` * reports that one asynchronously through `'error'`, so this has already answered `true`. * - * Waiting for the delivery's record to age out does NOT repair it, and an earlier version of - * this comment claimed it did. That record ages on the same cutoff the delivery's query uses, - * so "the record is gone" and "the payment is still in the read" exclude each other by - * construction — the retry it promised could never happen. The repair is in the `'error'` - * handler above, which drops the record so the next tick sends the state again. + * Waiting for the delivery's record to age out does NOT repair that. The record ages on the + * same cutoff the delivery's query uses, so "the record is gone" and "the payment is still in + * the read" exclude each other by construction — there is no later tick that would find both. + * The repair is in the `'error'` handler above, which drops the record so the next tick sends + * the state again. */ private sendMessage(device: PaymentDevice): boolean { const connections = this.clients.get(device.id); From 3199bbbaaeb80bcc8e1c48374638897c35fbe7a3 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:32:02 +0200 Subject: [PATCH 81/86] Make the lease owner name a RUN, and keep a switched-off job away from the table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The owner column is what renew and release match on, and it named the process. Two runs of one job can overlap inside a process: LockClass gives up on a run that outlives its declared timeout, and the next tick starts a second one. The second cannot take the claim while the first keeps renewing — but it can once a failed renewal has let the claim lapse, and from then on BOTH runs matched name + owner. The first to finish deleted the row the second was holding, and a third process could start the same money-moving job alongside it. Renewal had the mirror problem: the old run kept extending a claim it no longer had, and its own-ness check compared against itself. The owner is now minted per run (process part kept, so an operator still reads which container a row belongs to), and the in-flight record is keyed the same way — under a shared job-name key the second run replaced the first, and whichever finished deleted the entry of the other, leaving a run that shutdown neither waited for nor named. Also moved the DisabledProcess check OUTSIDE the lease: a job that is switched off must not claim anything. Inside, every tick of a disabled job still cost an INSERT and a DELETE on cron_lease — for the every-second jobs two statements a second each, and under DISABLED_PROCESSES='*' for every flagged job at once. And the renewal-margin comment now tells the truth: the interval bounds when the next attempt starts relative to the last ANSWER, so what the re-arm buys is one QUICK failure — an attempt that fails slowly spends the margin before it reports, and nothing bounds when that answer arrives. Each of the three behaviour changes is pinned by a test proven against its own removal. The shutdown test needed sharpening first: its original form stayed green with the keying reverted, which means it was not asking the question. --- .../__tests__/cron-lease.service.spec.ts | 120 ++++++++++++++++-- .../__tests__/dfx-cron.service.spec.ts | 49 +++++++ src/shared/services/cron-lease.service.ts | 102 ++++++++++----- src/shared/services/dfx-cron.service.ts | 33 +++-- 4 files changed, 248 insertions(+), 56 deletions(-) diff --git a/src/shared/services/__tests__/cron-lease.service.spec.ts b/src/shared/services/__tests__/cron-lease.service.spec.ts index 7d9b920234..2f37bbfd50 100644 --- a/src/shared/services/__tests__/cron-lease.service.spec.ts +++ b/src/shared/services/__tests__/cron-lease.service.spec.ts @@ -109,12 +109,16 @@ describe('CronLeaseService', () => { const run = service.run('SomeService::job', task); await settle(); - expect(service['inFlight'].has('SomeService::job')).toBe(true); + // Read by job rather than by key: the map is keyed per RUN, so two runs of one job stay + // apart in it. + const tracked = () => [...service['inFlight'].values()].map((entry) => entry.job); + + expect(tracked()).toEqual(['SomeService::job']); finish(); await run; - expect(service['inFlight'].has('SomeService::job')).toBe(false); + expect(tracked()).toEqual([]); }); it('does not start a lease-less run once shutdown has begun', async () => { @@ -188,7 +192,7 @@ describe('CronLeaseService', () => { // would happily take turns owning the same job. const { service, onQuery } = buildService({}); - await service.acquire('SomeService::job'); + await service.acquire('SomeService::job', 'worker:p:r'); const sql = (onQuery.mock.calls[0][0] as string).replace(/\s+/g, ' '); @@ -197,13 +201,13 @@ describe('CronLeaseService', () => { expect(sql).toContain('RETURNING "owner"'); }); - it('scopes renewal and release to this process', async () => { - // A run that already lost its lease must not be able to extend or delete the row a different - // process now owns. + it('scopes renewal and release to the run that took the claim', async () => { + // A run that already lost its lease must not be able to extend or delete the row another run + // now owns — in another process or, after LockClass gives up on a long one, in this one. const { service, onQuery } = buildService({}); - await service.renew('SomeService::job'); - await service.release('SomeService::job'); + await service.renew('SomeService::job', 'worker:p:r'); + await service.release('SomeService::job', 'worker:p:r'); const [renewSql] = onQuery.mock.calls[0]; const [releaseSql] = onQuery.mock.calls[1]; @@ -212,14 +216,82 @@ describe('CronLeaseService', () => { expect((releaseSql as string).replace(/\s+/g, ' ')).toContain('WHERE "name" = $1 AND "owner" = $2'); }); + it('gives two RUNS of the same job different owners, so one cannot release the claim of the other', async () => { + // Reachable in one process: `LockClass` gives up on a job that outlives its declared timeout, + // so the next tick starts a second run of the same job here while the first still works. It + // can take the claim once a failed renewal has let the first one lapse — and then, under a + // per-PROCESS owner, both runs would match `name + owner`. Whichever finished first would + // delete the row the other is holding, and a third process could start the job alongside it. + const { service, onQuery } = buildService({}); + + let release: (() => void) | undefined; + const first = service.run('SomeService::job', () => new Promise((resolve) => (release = resolve))); + await settle(); + + await service.run('SomeService::job', async () => undefined); + release?.(); + await first; + + const owners = onQuery.mock.calls + .filter(([sql]) => (sql as string).includes('INSERT INTO')) + .map(([, params]) => params[1] as string); + + expect(owners).toHaveLength(2); + expect(owners[0]).not.toEqual(owners[1]); + // Same process, so the process part is shared and only the run part differs — an operator + // still reads which container the row belongs to. + expect(owners[0].split(':').slice(0, 2)).toEqual(owners[1].split(':').slice(0, 2)); + + const deleted = onQuery.mock.calls + .filter(([sql]) => (sql as string).includes('DELETE FROM')) + .map(([, params]) => params[1] as string); + + expect(deleted).toEqual(expect.arrayContaining([owners[0], owners[1]])); + }); + + it('waits for BOTH runs of a job on shutdown, not just the last one to start', async () => { + // The in-flight record is keyed by run for the same reason. Keyed by job name the second run + // would replace the first, and whichever finished first would delete the entry of the other — + // leaving a run that shutdown neither waits for nor names. + const { service } = buildService({}); + + const done: string[] = []; + let releaseFirst: (() => void) | undefined; + let releaseSecond: (() => void) | undefined; + + const first = service + .run('SomeService::job', () => new Promise((resolve) => (releaseFirst = resolve))) + .then(() => done.push('first')); + await settle(); + const second = service + .run('SomeService::job', () => new Promise((resolve) => (releaseSecond = resolve))) + .then(() => done.push('second')); + await settle(); + + const shutdown = service.shutdown().then(() => done.push('shutdown')); + + // Only the SECOND run finishes. Whether shutdown is still waiting is the whole question: keyed + // by job name the first run's entry is already gone, so it would consider itself done here and + // `main.ts` would exit on top of a payout that is still running. + releaseSecond?.(); + await settle(); + + expect(done).toEqual(['second']); + + releaseFirst?.(); + await Promise.all([first, second, shutdown]); + + expect(done).toEqual(['second', 'first', 'shutdown']); + }); + it('gives two processes of the same role different owners', async () => { // The role alone would let a restarted container renew the lease its predecessor took. The // random part is what makes the owner identify a process rather than a kind of process. const { service: first, onQuery: firstQuery } = buildService({}); const { service: second, onQuery: secondQuery } = buildService({}); - await first.acquire('SomeService::job'); - await second.acquire('SomeService::job'); + await first.run('SomeService::job', async () => undefined); + await second.run('SomeService::job', async () => undefined); const firstOwner = firstQuery.mock.calls[0][1][1] as string; const secondOwner = secondQuery.mock.calls[0][1][1] as string; @@ -237,7 +309,7 @@ describe('CronLeaseService', () => { it('claims for a minute, whatever the job it guards is allowed to take', async () => { const { service, onQuery } = buildService({}); - await service.acquire('SomeService::job'); + await service.acquire('SomeService::job', 'worker:p:r'); expect(onQuery.mock.calls[0][1][2]).toEqual('60'); }); @@ -245,7 +317,7 @@ describe('CronLeaseService', () => { it('renews for the same short span', async () => { const { service, onQuery } = buildService({}); - await service.renew('SomeService::job'); + await service.renew('SomeService::job', 'worker:p:r'); expect(onQuery.mock.calls[0][1][2]).toEqual('60'); }); @@ -406,6 +478,26 @@ describe('CronLeaseService', () => { expect(main).toMatch(/leases\s*\n?\s*\.shutdown\(\)/); }); + it('stops taking new requests before it waits', () => { + // The wait keeps this process alive for up to the grace period. Without closing the listener + // it goes on accepting requests for that whole span and then cuts them off at `process.exit` + // — a window that did not exist while the signal ended the process at once. Comment lines + // are dropped first, as in the test below: the reason sits next to the call. + const main = readFileSync(join(__dirname, '..', '..', '..', 'main.ts'), 'utf8') + .split('\n') + .filter((line) => !/^\s*(\/\/|\*|\/\*)/.test(line)) + .join('\n'); + + const handler = main.slice(main.indexOf('function releaseCronLeasesOnShutdown')); + const close = handler.indexOf('app.getHttpServer().close()'); + const wait = handler.indexOf('.shutdown()'); + + expect(close).toBeGreaterThan(-1); + expect(close).toBeLessThan(wait); + // And the listener only: `app.close()` runs the module-destroy hooks the test below is about. + expect(handler).not.toMatch(/\bapp\.close\(/); + }); + it("is NOT wired through Nest's global shutdown hooks", () => { // `enableShutdownHooks` would also start running nine `onModuleDestroy` hooks that have // never run here, before this one, emptying the strategy registries that PayIn, PayOut and @@ -520,7 +612,7 @@ describe('CronLeaseService', () => { expect(service.takeFailures().healthy).toBe(false); onQuery.mockResolvedValue([{ owner: 'worker:1' }]); - await service.acquire('SomeService::job'); + await service.acquire('SomeService::job', 'worker:p:r'); expect(service.takeFailures().healthy).toBe(true); }); @@ -537,7 +629,7 @@ describe('CronLeaseService', () => { expect(service.takeFailures().healthy).toBe(false); onQuery.mockResolvedValue([[], 1]); - await service.renew('SomeService::job'); + await service.renew('SomeService::job', 'worker:p:r'); expect(service.takeFailures().healthy).toBe(true); }); diff --git a/src/shared/services/__tests__/dfx-cron.service.spec.ts b/src/shared/services/__tests__/dfx-cron.service.spec.ts index 0beb4db143..1a81e692de 100644 --- a/src/shared/services/__tests__/dfx-cron.service.spec.ts +++ b/src/shared/services/__tests__/dfx-cron.service.spec.ts @@ -13,6 +13,7 @@ import { CronJob } from 'cron'; import { DataSource } from 'typeorm'; import { CronLeaseService } from '../cron-lease.service'; import { DfxCronService } from '../dfx-cron.service'; +import * as ProcessService from '../process.service'; import { Process } from '../process.service'; /** Builds a provider instance carrying @DfxCron metadata, as the decorator would. */ @@ -227,6 +228,54 @@ describe('DfxCronService', () => { expect(leased).not.toContain('Object::bothJob'); }); + it('does not claim the lease for a job that is switched off', async () => { + // A job that is off must not touch the table. Inside the lease the disabled check still cost + // one INSERT and one DELETE per tick — for the jobs that tick every second, two statements a + // second each, and under DISABLED_PROCESSES='*' for every flagged job at once. + process.env.CRON_ROLE = 'worker'; + new ConfigService(GetConfig()); + + const disabled = jest.spyOn(ProcessService, 'DisabledProcess').mockReturnValue(true); + + const seen: string[] = []; + const body = jest.fn(); + const instance = { flaggedJob: body }; + Reflect.defineMetadata( + DFX_CRONJOB_PARAMS, + { + expression: CronExpression.EVERY_MINUTE, + scope: CronScope.WORKER, + process: Process.MONITOR_EVENT_LOOP, + useDelay: false, + } as DfxCronParams, + instance.flaggedJob, + ); + + const discovery = createMock({ + getProviders: () => + [{ instance, isDependencyTreeStatic: () => true }] as ReturnType, + }); + const metadataScanner = createMock({ getAllMethodNames: (i: object) => Object.keys(i) }); + const leaseSpy = createMock({ + run: (job: string, task: () => Promise) => { + seen.push(job); + return task(); + }, + takeFailures: () => ({ healthy: true, count: 0 }), + }); + + new DfxCronService(discovery, metadataScanner, createMock(), leaseSpy).onModuleInit(); + + const scheduled = (CronJob as unknown as jest.Mock).mock.calls.map(([, fn]) => fn as () => unknown); + for (const fire of scheduled) await fire(); + + expect(disabled).toHaveBeenCalled(); + expect(seen).toEqual([]); + expect(body).not.toHaveBeenCalled(); + + disabled.mockRestore(); + }); + it('does not turn a long job timeout into a long lease', async () => { // The lease used to expire when the job's own timeout did. `timeout` is in seconds, per // LockClass, so the 7200 declared below left the row behind for two hours after a process diff --git a/src/shared/services/cron-lease.service.ts b/src/shared/services/cron-lease.service.ts index b9295858bd..977e59f91f 100644 --- a/src/shared/services/cron-lease.service.ts +++ b/src/shared/services/cron-lease.service.ts @@ -25,10 +25,18 @@ const LEASE_TTL_SECONDS = 60; * Renew at a third of the lease. * * The timer below re-arms only once the previous attempt has settled, so the attempts fall at 20 s - * and then 20 s after each answer — never earlier, and later whenever the database is slow. One - * failed renewal therefore still leaves a further attempt with roughly 20 s to spare; two do not, - * because the third attempt starts at 60 s at the earliest, which is the moment the claim lapses. - * The margin this buys is one lost renewal, not two. + * and then 20 s after each answer — never earlier, and later whenever the database is slow. + * + * What that buys is one QUICK failure, not one failure. An attempt that fails at once at 20 s puts + * the next at 40 s, still 20 s inside the lease. An attempt that fails SLOWLY spends the margin + * before it reports: one that comes back at 45 s puts the next at 65 s, five seconds after the + * claim has already lapsed. The interval bounds when the next attempt starts relative to the last + * ANSWER, and nothing here bounds when that answer arrives — there is no statement timeout on + * these queries. + * + * That is the honest shape of the margin, and it is why the expiry is not read as a guarantee + * anywhere: a slow database is precisely the case where two processes can end up running the same + * job, and the section "What it does not do" below says so. */ const RENEWAL_INTERVAL_MS = (LEASE_TTL_SECONDS / 3) * 1000; @@ -86,13 +94,18 @@ export class CronLeaseService implements OnModuleInit { private ownerId?: string; /** - * The runs this process currently holds a lease for, by job name. + * The runs this process has started and not yet finished, by the run's own owner string. * * Kept so shutdown knows what is still outstanding. The stored promise has its rejection already * absorbed: the job's own error belongs to its caller, and a second consumer of the same * rejection here would surface as an unhandled one. + * + * Keyed by RUN, not by job name, for the same reason the owner is: two runs of one job can + * overlap in this process once `LockClass` has given up on the first. Under a shared key the + * second would replace the first here, and whichever finished first would then delete the + * entry of the other — leaving a run that `shutdown` neither waits for nor names. */ - private readonly inFlight = new Map>(); + private readonly inFlight = new Map }>(); /** * Whether the last lease operation reached the table. Sticky until one succeeds, so a role whose @@ -122,10 +135,29 @@ export class CronLeaseService implements OnModuleInit { * Resolved on first use rather than in a field initializer: `Config` does not exist until * `ConfigService` has been constructed, and a provider built before it would take down the boot. */ - private get owner(): string { + private get process(): string { return (this.ownerId ??= `${Config.cronRole}:${randomUUID()}`); } + /** + * Identifies ONE RUN. The owner written to the table is per run, not per process. + * + * Per process it would have been one identity for two runs that can overlap inside it. That + * overlap is reachable: `LockClass` gives up on a job that outlives its declared timeout, so the + * next tick starts a second run of the same job HERE while the first is still working. It cannot + * take the claim while the first keeps renewing — but it can once a failed renewal has let the + * claim lapse, and then both runs match `name + owner`. The first one to finish would delete the + * row the second is holding, and a third process could start the job alongside it. Renewal has + * the mirror problem: the old run would keep extending a claim it no longer has, and its + * `stillOurs` check would come back true because it is comparing against itself. + * + * With one identity per run, both statements name the run that took the claim. The old run's + * renewal returns false and says so, and its release matches nothing. + */ + private newOwner(): string { + return `${this.process}:${randomUUID()}`; + } + constructor(private readonly dataSource: DataSource) {} /** @@ -174,7 +206,7 @@ export class CronLeaseService implements OnModuleInit { * them sees a returned row. Read the `WHERE` as "only if nobody is currently holding it" — an * unexpired row belonging to another process leaves the update out and returns nothing. */ - async acquire(job: string): Promise { + async acquire(job: string, owner: string): Promise { const claimed = await this.dataSource.query( `INSERT INTO "cron_lease" ("name", "owner", "acquired", "expires") VALUES ($1, $2, now(), now() + ($3 || ' seconds')::interval) @@ -182,7 +214,7 @@ export class CronLeaseService implements OnModuleInit { SET "owner" = EXCLUDED."owner", "acquired" = EXCLUDED."acquired", "expires" = EXCLUDED."expires" WHERE "cron_lease"."expires" <= now() RETURNING "owner"`, - [job, this.owner, `${LEASE_TTL_SECONDS}`], + [job, owner, `${LEASE_TTL_SECONDS}`], ); this.recordSuccess(); @@ -191,16 +223,16 @@ export class CronLeaseService implements OnModuleInit { } /** - * Pushes the expiry out while the job is still running. Returns false when this process is no - * longer the owner — which means another process has taken the job over and this run should be - * treated as having lost its claim. + * Pushes the expiry out while the job is still running. Returns false when this RUN is no longer + * the owner — which means the claim has lapsed and someone else has taken the job over, and this + * run should be treated as having lost it. */ - async renew(job: string): Promise { + async renew(job: string, owner: string): Promise { const [, affected] = await this.dataSource.query( `UPDATE "cron_lease" SET "expires" = now() + ($3 || ' seconds')::interval WHERE "name" = $1 AND "owner" = $2`, - [job, this.owner, `${LEASE_TTL_SECONDS}`], + [job, owner, `${LEASE_TTL_SECONDS}`], ); this.recordSuccess(); @@ -209,11 +241,11 @@ export class CronLeaseService implements OnModuleInit { } /** - * Releases the lease. Scoped to this owner so a run that already lost the lease cannot delete - * the row a different process is now holding. + * Releases the lease. Scoped to the owner that took it, so a run which already lost the lease + * cannot delete the row another run is now holding — in another process or in this one. */ - async release(job: string): Promise { - await this.dataSource.query(`DELETE FROM "cron_lease" WHERE "name" = $1 AND "owner" = $2`, [job, this.owner]); + async release(job: string, owner: string): Promise { + await this.dataSource.query(`DELETE FROM "cron_lease" WHERE "name" = $1 AND "owner" = $2`, [job, owner]); this.recordSuccess(); } @@ -243,9 +275,11 @@ export class CronLeaseService implements OnModuleInit { // releases its lease, and, more importantly, part-way through whatever it was doing. if (this.shuttingDown) return; + const owner = this.newOwner(); + let acquired: boolean; try { - acquired = await this.acquire(job); + acquired = await this.acquire(job, owner); } catch (e) { this.recordFailure(e); @@ -281,7 +315,7 @@ export class CronLeaseService implements OnModuleInit { // absent from the "still running" warning — and it would start even if shutdown had begun // during the failed claim, which is a longer window than the healthy one because the // attempt runs to the database timeout. - return this.track(job, task); + return this.track(job, owner, task); } if (!acquired) { @@ -302,12 +336,12 @@ export class CronLeaseService implements OnModuleInit { // yet, so `shutdown` does not wait for it, and the process can exit before the release runs. // What then remains is a claim nobody holds — it lapses within the TTL like any other, which // is the bound that always applies. The release only ever shortens that wait. - const renewal = this.keepAlive(job); + const renewal = this.keepAlive(job, owner); - return this.track(job, task, () => { + return this.track(job, owner, task, () => { renewal.stop(); - return this.release(job).catch((e) => { + return this.release(job, owner).catch((e) => { this.recordFailure(e); this.logger.error(`Could not release the lease for ${job}`, e); }); @@ -325,7 +359,12 @@ export class CronLeaseService implements OnModuleInit { * `after` is what the claim-holding path adds: stop renewing, hand the claim back. The * lease-less path has nothing to hand back. */ - private async track(job: string, task: () => Promise, after?: () => Promise): Promise { + private async track( + job: string, + owner: string, + task: () => Promise, + after?: () => Promise, + ): Promise { // As late as possible, because the step before it is a round trip: shutdown can begin while a // claim is being taken, or while a failing attempt runs to its timeout. A run started after // that point is not in the set `shutdown` waits on and would be cut off part-way through. @@ -339,14 +378,11 @@ export class CronLeaseService implements OnModuleInit { await task(); } finally { await after?.(); - this.inFlight.delete(job); + this.inFlight.delete(owner); } })(); - this.inFlight.set( - job, - run.catch(() => undefined), - ); + this.inFlight.set(owner, { job, run: run.catch(() => undefined) }); return run; } @@ -383,14 +419,14 @@ export class CronLeaseService implements OnModuleInit { // for and would be cut off part-way through — see the guard at the top of `run`. this.shuttingDown = true; - const running = [...this.inFlight.values()]; + const running = [...this.inFlight.values()].map((entry) => entry.run); if (!running.length) return; this.logger.info(`Shutting down: waiting up to ${SHUTDOWN_GRACE_MS / 1000}s for ${running.length} running job(s)`); await Promise.race([Promise.all(running), this.shutdownGrace()]); - const stranded = [...this.inFlight.keys()]; + const stranded = [...this.inFlight.values()].map((entry) => entry.job); if (stranded.length) this.logger.warn( `Shutting down with ${stranded.length} job(s) still running (${stranded.join(', ')}); ` + @@ -430,7 +466,7 @@ export class CronLeaseService implements OnModuleInit { * Losing the claim does not stop the run. There is nothing here that could stop it, and the * timer deliberately keeps going: this process holds the claim for as long as it can renew it. */ - private keepAlive(job: string): { stop: () => void } { + private keepAlive(job: string, owner: string): { stop: () => void } { let stopped = false; let timer: NodeJS.Timeout; @@ -438,7 +474,7 @@ export class CronLeaseService implements OnModuleInit { // Unref'd: a pending timer must never hold the process open on shutdown. timer = setTimeout(async () => { try { - const stillOurs = await this.renew(job); + const stillOurs = await this.renew(job, owner); if (!stillOurs) this.logger.error(`Lost the lease for ${job} while it was still running`); } catch (e) { this.recordFailure(e); diff --git a/src/shared/services/dfx-cron.service.ts b/src/shared/services/dfx-cron.service.ts index a7f3743a7e..f47be257d6 100644 --- a/src/shared/services/dfx-cron.service.ts +++ b/src/shared/services/dfx-cron.service.ts @@ -208,22 +208,37 @@ export class DfxCronService implements OnModuleInit { private guardAcrossProcesses(cronJobName: string, data: CronJobData): () => Promise { const task = this.wrapFunction(data); - if (data.params.scope === CronScope.BOTH) return task; - - return () => this.leases.run(cronJobName, task, data.params.scope === CronScope.API); + const leased = + data.params.scope === CronScope.BOTH + ? task + : () => this.leases.run(cronJobName, task, data.params.scope === CronScope.API); + + // OUTSIDE the lease, and that order is the point: a job that is switched off must not claim + // anything. Inside, every tick of a disabled job still cost one INSERT and one DELETE on + // `cron_lease` — for the jobs that tick every second, two statements a second each, and under + // `DISABLED_PROCESSES='*'` for all of them at once. That is dead tuples on a table whose whole + // purpose is to be read quickly, produced by jobs that are doing nothing. + return this.skipWhenDisabled(leased, data); } - private wrapFunction(data: CronJobData) { + private skipWhenDisabled(task: () => Promise, data: CronJobData): () => Promise { + const { process } = data.params; + if (!process) return task; + const context = { target: data.instance.constructor.name, method: data.methodName }; - return async (...args: any) => { - if (data.params.process && DisabledProcess(data.params.process)) { - this.logger.verbose( - `Skipping ${context.target}::${context.method} - process ${data.params.process} is disabled`, - ); + return async () => { + if (DisabledProcess(process)) { + this.logger.verbose(`Skipping ${context.target}::${context.method} - process ${process} is disabled`); return; } + return task(); + }; + } + + private wrapFunction(data: CronJobData) { + return async (...args: any) => { if (data.params.useDelay ?? true) await this.cronJobDelay(data.params.expression); await data.methodRef.apply(data.instance, args); From 456359a21651ae7bea303c7581f3facfcaee5c71 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:32:03 +0200 Subject: [PATCH 82/86] Forget deliveries for a peer the ping sweep drops, and stop overstating two bounds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sweep is the third way a device command can be lost after the sink answered true, and the only one that produces neither a throw nor an 'error' event: the peer stopped answering without closing anything, so a send between the last two sweeps went into a socket nobody was reading. The delivery marker stayed put and the device was never told again — not even on reconnect. The sweep now drops the device's delivery record along with its sockets. An orderly 'close' still does NOT: a peer that completes the closing handshake was reading until it did. Two comments claimed more than the code holds. The index migration asserted a device has "a handful" of payments — a statement about production data; what the index actually removes is the scan of every OTHER device's history. And the delivery-record comment claimed an hour-shaped bound, while the bound is the read itself: expiryDate comes from the caller, validated only as a date, so a merchant setting it far ahead keeps payments in the read — and in the record — until then. Whether to cap it is a contract question, recorded as an open operator decision, not decided here. The wait-timeout branch now also says WHY it answers with the payment as handed in instead of re-reading: the callers time out together, and a re-read per timing-out caller would turn one batched read every 15 s into a burst of single-row reads. --- ...0000-AddPaymentLinkPaymentDeviceIdIndex.js | 7 ++-- .../controllers/payment-link.gateway.ts | 12 +++++- .../services/payment-link-payment.service.ts | 39 +++++++++++++------ 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/migration/1785620000000-AddPaymentLinkPaymentDeviceIdIndex.js b/migration/1785620000000-AddPaymentLinkPaymentDeviceIdIndex.js index 669dc79407..9d1cccfa91 100644 --- a/migration/1785620000000-AddPaymentLinkPaymentDeviceIdIndex.js +++ b/migration/1785620000000-AddPaymentLinkPaymentDeviceIdIndex.js @@ -13,9 +13,10 @@ * ticks for as long as a device stays connected. * * A single-column index is enough for the shape of the query. It filters `deviceId IN (…)` on the - * handful of devices connected to this process and narrows the result further by `expiryDate` and - * by status — but a device has few payments, so the index gets the row count down to that handful - * before the remaining conditions are applied. + * handful of devices connected to this process; `expiryDate` and the status conditions are applied + * to what that yields and are not part of the index. What the index removes is the scan of every + * OTHER device's payments, which is the part that grows with the table. What it leaves is one + * device's own history — bounded by how much that terminal has taken, not by the query. * * The name is the deterministic one TypeORM's `DefaultNamingStrategy` derives, since CONTRIBUTING * disallows custom index names: `IDX_` followed by the first 26 hex characters of diff --git a/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts b/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts index 9d530c3823..ea772ecaaf 100644 --- a/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts +++ b/src/subdomains/core/payment-link/controllers/payment-link.gateway.ts @@ -97,6 +97,15 @@ export class PaymentLinkGateway implements OnGatewayConnection, OnModuleInit { // on the next sweep, and on every one after that. this.removeClient(device, clientId); connection.socket.terminate(); + + // And what this process believes it told that device goes with it. This is the third + // way a command can be lost after the sink answered `true`, and the only one that + // produces neither a throw nor an `'error'`: the peer stopped answering without closing + // anything, so a `send` between the last two sweeps went into a socket nobody was + // reading. An orderly `'close'` is different and deliberately does NOT do this — there + // the peer completed the closing handshake, which means it was still reading, and + // forgetting would repeat every command on every reconnect. + this.paymentService.forgetDeliveries(device); continue; } @@ -120,7 +129,8 @@ export class PaymentLinkGateway implements OnGatewayConnection, OnModuleInit { // `error` does one thing more than `close`: it is how `ws` reports a send on a socket that // closed between the delivery's state check and the call itself. The delivery has already // recorded that command as sent, so the record has to go — see - // PaymentLinkPaymentService.forgetDeliveries for why `close` must NOT do the same. + // PaymentLinkPaymentService.forgetDeliveries for why `close` must NOT do the same, and + // `checkConnections` above for the third case, which reaches neither handler. client.on('error', () => { this.removeClient(device, clientId); this.paymentService.forgetDeliveries(device); diff --git a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts index ab56b4bc75..5b2b8679e9 100644 --- a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts @@ -328,8 +328,16 @@ export class PaymentLinkPaymentService { try { return await this.paymentWaitMap.wait(payment.id, PAYMENT_WAIT_TIMEOUT_SECONDS * 1000); } catch { - // The wait elapsed. Answer with the payment as it stands rather than fail: the caller asked - // whether anything happened, and "not within this window" is an answer to that. + // The wait elapsed. Answer with the payment AS THE CALLER HANDED IT IN rather than fail: + // "nothing was observed within this window" is an answer to what it asked, and the caller + // polls again. + // + // Deliberately not re-read here, although that would close a gap of up to one tick: a change + // in the last 15 s before the timeout has not been picked up yet, so this can answer + // `Pending` for a payment that just completed. Re-reading would cost one query PER TIMING-OUT + // CALLER, and they time out together — a shop with many terminals would turn one batched read + // every 15 s into a burst of single-row reads. The tick path already answers all of them with + // one query, and the next poll is at most a round trip away. return payment; } finally { // The waiter owns its entry. `resolveWaiters` clears it on the delivery path; this clears it @@ -417,9 +425,15 @@ export class PaymentLinkPaymentService { /** * Drops what the read can no longer return. An entry is kept while its payment is still asked * for, NOT while its device is connected: a device that reconnects finds its record intact and is - * not told a second time about what it already heard. That is also what bounds the record — - * entries age out on the same cutoff the query uses, and a device whose entries have all gone - * leaves with them. + * not told a second time about what it already heard. The record therefore holds exactly what + * the read can still produce, and a device whose entries have all gone leaves with them. + * + * That ties the size of this map to the READ, not to a span: `expiryDate` comes from the caller + * (`CreatePaymentLinkPaymentDto`, optional and validated only as a date) and falls back to the + * link's `paymentTimeout` when it is left out. For the payments the API dates itself that means + * roughly the timeout plus an hour; a caller that sets an expiry far ahead keeps its payments in + * the read — and here — until then. Nothing in this class caps that, and capping it would be a + * change to what a merchant may ask for rather than to this delivery. */ private pruneDeliveries(cutoff: Date): void { for (const [deviceId, delivered] of this.deviceDeliveries) { @@ -433,14 +447,15 @@ export class PaymentLinkPaymentService { /** * Drops what this process believes it told a device, so the next tick tells it again. * - * Called by the gateway when a socket reports `'error'` — the one signal that a command may - * have failed AFTER the sink answered `true`. `ws` raises a send on a socket that closed - * mid-call that way, and nothing else can see it: the state was open when it was read, and the - * call did not throw. + * Called by the gateway on the two signals that a command may have failed AFTER the sink + * answered `true`, both of which are invisible here: `'error'`, which is how `ws` reports a send + * on a socket that closed mid-call (the state was open when it was read, and the call did not + * throw), and the ping sweep dropping a peer that stopped answering without closing anything — + * there a send simply went into a socket nobody was reading. * - * Deliberately not called on `'close'`. An orderly close is not evidence that anything failed, - * and forgetting there would repeat every command on every reconnect — which is what the record - * exists to prevent. + * Deliberately NOT called on `'close'`. A peer that completes the closing handshake was reading + * until it did, so an orderly close is not evidence that anything failed; forgetting there would + * repeat every command on every reconnect, which is what the record exists to prevent. */ forgetDeliveries(deviceId: string): void { this.deviceDeliveries.delete(deviceId); From 16ac436a081b01264a8e00d946daebc157ad3b7b Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:32:05 +0200 Subject: [PATCH 83/86] Load the statistic on demand, drop the revocation kill switch, close the listener first MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit StatisticService broke the rule this PR introduces: its job is api-scoped AND leased, so among several API processes one wins each tick — and getAll served a process-local field nothing else filled. A second API process (a blue-green window, a --scale) would have answered with its boot numbers for the rest of its life. getAll now refreshes what is missing or older than the job's own hour, with concurrent readers sharing one refresh, same shape as the fee service. JwtRevocationSyncService loses the process flag this branch had given it. The job fills the denylist of blocked accounts; switched off it does not empty that list, it stops writing to it — accounts blocked afterwards keep their live JWTs, and nothing reports the state. Its sibling syncStaffKycClearance stays flag-less for exactly this reason and now both say so. And the shutdown handler stops taking new connections before it waits: the grace period keeps the process alive for up to 10 s after SIGTERM, and without closing the listener it accepted requests for that whole span and then cut them off at process.exit — a window that did not exist while the signal ended the process at once. The listener only, deliberately not app.close(), which would run the module-destroy hooks the handler exists to avoid. --- src/main.ts | 16 ++++ src/shared/services/process.service.ts | 1 - .../__tests__/statistic.service.spec.ts | 74 +++++++++++++++++++ .../core/statistic/statistic.service.ts | 47 +++++++++++- .../user-data/jwt-revocation-sync.service.ts | 14 ++-- 5 files changed, 141 insertions(+), 11 deletions(-) diff --git a/src/main.ts b/src/main.ts index 2970b6f41f..f4c4b1a5b4 100644 --- a/src/main.ts +++ b/src/main.ts @@ -150,6 +150,10 @@ async function bootstrap() { * Registering a handler means Node no longer terminates on the signal by itself, so this has to * exit. `CronLeaseService.shutdown` is bounded by its own grace period, and a second signal takes * the impatient path — otherwise a stuck shutdown would hold the container until SIGKILL. + * + * It also means the process outlives the signal, which is new. Everything that was true only + * because the process ended immediately has to be re-established explicitly — starting with not + * accepting further requests; see the listener close below. */ function releaseCronLeasesOnShutdown(app: INestApplication): void { const logger = new DfxLogger('Shutdown'); @@ -167,6 +171,18 @@ function releaseCronLeasesOnShutdown(app: INestApplication): void { started = true; logger.info(`${signal} received, releasing the cron leases`); + // Stop taking NEW connections first. Waiting for the running jobs keeps this process alive + // for up to the grace period, and without this it would go on accepting requests for that + // whole span and then cut them off mid-flight at the `process.exit` below — a window that + // did not exist while the signal ended the process at once. + // + // The listener only, NOT `app.close()`: that is the call which runs the nine + // `onModuleDestroy` hooks described above, and it would empty the strategy registries out + // from under the jobs this wait exists to protect. Not awaited either — a keep-alive + // connection can hold the callback back indefinitely, and the bound here is the grace + // period, not the client. + app.getHttpServer().close(); + void leases .shutdown() .catch((e) => logger.error('Failed to release the cron leases on shutdown:', e)) diff --git a/src/shared/services/process.service.ts b/src/shared/services/process.service.ts index d00d31deaa..c1e0ad1609 100644 --- a/src/shared/services/process.service.ts +++ b/src/shared/services/process.service.ts @@ -120,7 +120,6 @@ export enum Process { LEDGER_COA_BOOTSTRAP = 'LedgerCoaBootstrap', DEX_PURCHASE_ORDER = 'DexPurchaseOrder', REF_CLEANUP = 'RefCleanup', - JWT_REVOCATION_SYNC = 'JwtRevocationSync', LATEST_BALANCE_CACHE = 'LatestBalanceCache', SPARK_TOKEN_OPTIMIZATION = 'SparkTokenOptimization', } diff --git a/src/subdomains/core/statistic/__tests__/statistic.service.spec.ts b/src/subdomains/core/statistic/__tests__/statistic.service.spec.ts index ce40890fca..5afe171716 100644 --- a/src/subdomains/core/statistic/__tests__/statistic.service.spec.ts +++ b/src/subdomains/core/statistic/__tests__/statistic.service.spec.ts @@ -55,6 +55,80 @@ describe('StatisticService', () => { }); }); + /** + * `doUpdate` is scoped `api` AND leased, so it runs in ONE api process per tick. Everything the + * request path reads therefore has to be fillable from the request path — CONTRIBUTING: "A cron + * job may refresh it, but must not be the only thing filling it." + */ + describe('getAll', () => { + it('fills the statistic when there is none, instead of answering with nothing', async () => { + const update = jest.spyOn(service, 'doUpdate').mockResolvedValue(undefined); + + await service.getAll(); + + expect(update).toHaveBeenCalledTimes(1); + }); + + it('serves what is on hand without asking again', async () => { + service['statistic'] = { at: new Date(), data: { totalVolume: { buy: 1, sell: 2 } } as never }; + const update = jest.spyOn(service, 'doUpdate').mockResolvedValue(undefined); + + await expect(service.getAll()).resolves.toEqual({ totalVolume: { buy: 1, sell: 2 } }); + expect(update).not.toHaveBeenCalled(); + }); + + it('refreshes what has aged past the job it complements', async () => { + // A second api process — a blue-green window, a `--scale` — loses the lease on every tick + // and would otherwise serve what it read at boot for the rest of its life. + service['statistic'] = { at: new Date(Date.now() - 61 * 60 * 1000), data: {} as never }; + const update = jest.spyOn(service, 'doUpdate').mockResolvedValue(undefined); + + await service.getAll(); + + expect(update).toHaveBeenCalledTimes(1); + }); + + it('starts ONE refresh for concurrent readers', async () => { + // The field is written when a refresh resolves, so without the in-flight promise every + // request arriving until then starts its own — and one refresh is four aggregations over + // the whole volume history. + let finish: () => void; + const update = jest + .spyOn(service, 'doUpdate') + .mockImplementation(() => new Promise((resolve) => (finish = resolve))); + + const calls = [service.getAll(), service.getAll(), service.getAll()]; + await new Promise((resolve) => setImmediate(resolve)); + finish(); + await Promise.all(calls); + + expect(update).toHaveBeenCalledTimes(1); + }); + + it('answers with what it has when the refresh fails', async () => { + // The endpoint is public and read-only: an aggregation that cannot run right now says + // nothing about the numbers already here. + service['statistic'] = { at: new Date(Date.now() - 61 * 60 * 1000), data: { old: true } as never }; + jest.spyOn(service, 'doUpdate').mockRejectedValue(new Error('aggregation failed')); + jest.spyOn(service['logger'], 'error').mockImplementation(); + + await expect(service.getAll()).resolves.toEqual({ old: true }); + }); + + it('lets the next reader retry after a failed refresh', async () => { + const update = jest + .spyOn(service, 'doUpdate') + .mockRejectedValueOnce(new Error('aggregation failed')) + .mockResolvedValue(undefined); + jest.spyOn(service['logger'], 'error').mockImplementation(); + + await service.getAll(); + await service.getAll(); + + expect(update).toHaveBeenCalledTimes(2); + }); + }); + /** * The start-up fill runs outside the scheduler, so none of the conditions the scheduler applies * to `doUpdate` reached it: not the scope, not the process flag, not any error handling. Each diff --git a/src/subdomains/core/statistic/statistic.service.ts b/src/subdomains/core/statistic/statistic.service.ts index aa86dbf0a8..6628eb13a6 100644 --- a/src/subdomains/core/statistic/statistic.service.ts +++ b/src/subdomains/core/statistic/statistic.service.ts @@ -15,7 +15,19 @@ import { SettingStatus, StatisticDto } from './dto/statistic.dto'; export class StatisticService implements OnModuleInit { private readonly logger = new DfxLogger(StatisticService); - private statistic: StatisticDto; + /** How long a filled statistic is served before a reader refreshes it; matches the job's own hour. */ + private static readonly MAX_AGE_SECONDS = 60 * 60; + + private statistic?: { at: Date; data: StatisticDto }; + + /** + * The refresh currently in flight, so concurrent readers of a cold or stale statistic share one. + * + * The field is only written when a refresh RESOLVES, so without this every request arriving + * until then would start its own — and one refresh is four aggregations over the whole volume + * history, a job that declares `timeout: 7200` for a reason. + */ + private refreshing?: Promise; constructor( private readonly buyService: BuyService, @@ -49,7 +61,7 @@ export class StatisticService implements OnModuleInit { @DfxCron(CronExpression.EVERY_HOUR, { scope: CronScope.API, process: Process.UPDATE_STATISTIC, timeout: 7200 }) async doUpdate(): Promise { - this.statistic = { + const data: StatisticDto = { totalVolume: { buy: Util.round(await this.buyService.getTotalVolume(), Config.defaultVolumeDecimal), sell: Util.round(await this.sellService.getTotalVolume(), Config.defaultVolumeDecimal), @@ -60,6 +72,10 @@ export class StatisticService implements OnModuleInit { }, status: await this.getStatus(), }; + + // Assigned as a whole, once every field is in. Written field by field, a reader arriving + // mid-refresh would see a statistic whose volumes and status come from different moments. + this.statistic = { at: new Date(), data }; } async getStatus(): Promise { @@ -67,7 +83,30 @@ export class StatisticService implements OnModuleInit { return settings.reduce((prev, curr) => ({ ...prev, [curr.key.replace('Status', '')]: curr.value }), {}); } - getAll(): StatisticDto { - return this.statistic; + /** + * Serves the statistic, refreshing it here when what is on hand is missing or older than the + * job's own interval. + * + * CONTRIBUTING: "A cron job may refresh it, but must not be the only thing filling it." The job + * above refreshes; this is the load, and it is what the lease made necessary. Scoped `api` AND + * leased, the job runs in ONE api process per tick — a second one (a blue-green window, a + * `--scale`) would otherwise serve whatever it read at boot for the rest of its life, with no + * later tick ever reaching it. The `DisabledProcess` switch has the same effect on the process + * that holds the lease. + */ + async getAll(): Promise { + const fresh = this.statistic && Util.secondsDiff(this.statistic.at) <= StatisticService.MAX_AGE_SECONDS; + + if (!fresh) { + try { + await (this.refreshing ??= this.doUpdate().finally(() => (this.refreshing = undefined))); + } catch (e) { + // What is on hand beats failing the request: this endpoint is public and read-only, and + // an aggregation that cannot run right now says nothing about the numbers already here. + this.logger.error('Failed to refresh the statistic on demand:', e); + } + } + + return this.statistic?.data; } } diff --git a/src/subdomains/generic/user/models/user-data/jwt-revocation-sync.service.ts b/src/subdomains/generic/user/models/user-data/jwt-revocation-sync.service.ts index a333eb5e3d..23ce2fad16 100644 --- a/src/subdomains/generic/user/models/user-data/jwt-revocation-sync.service.ts +++ b/src/subdomains/generic/user/models/user-data/jwt-revocation-sync.service.ts @@ -1,7 +1,6 @@ import { Injectable } from '@nestjs/common'; import { CronExpression } from '@nestjs/schedule'; import { SettingService } from 'src/shared/models/setting/setting.service'; -import { Process } from 'src/shared/services/process.service'; import { CronScope, DfxCron } from 'src/shared/utils/cron'; import { In } from 'typeorm'; import { RiskStatus, UserDataStatus } from './user-data.enum'; @@ -26,11 +25,14 @@ export class JwtRevocationSyncService { // Runs every minute: fast revocation of a blocked or compromised account is a security requirement that // warrants the security-revocation exception to the "prefer 15min" cron guideline. - @DfxCron(CronExpression.EVERY_MINUTE, { - scope: CronScope.WORKER, - process: Process.JWT_REVOCATION_SYNC, - timeout: 1800, - }) + // + // And deliberately WITHOUT a `process` flag, for the same reason and matching + // StaffKycClearanceService::syncStaffKycClearance, which states it there. A flag is a switch that + // turns the job off in the database, `DISABLED_PROCESSES='*'` turns off everything that has one, + // and switched off this job does not empty the auto denylist — it stops writing to it. Accounts + // blocked after that keep their live JWTs, and nothing reports it: the role heartbeat goes on + // saying `lease ok`. A switch whose use is silent does not belong on a revocation path. + @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, timeout: 1800 }) async syncDeniedJwtAccounts(): Promise { const blockedAccounts = await this.userDataRepo.find({ select: { id: true }, From d8bc7a2c3fb7343dd0e22e6915071c7fd9f1e23e Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:32:06 +0200 Subject: [PATCH 84/86] Prove the lease over every interleaving instead of the ones somebody thought of MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cron-lease.protocol.spec.ts enumerates ALL orders in which two runs can issue their statements, with and without a lapse in between, and checks the owner invariants over each one: only the run that took a claim can extend or delete it, a second acquire succeeds exactly when the first holder released or lapsed, and an orphaned claim is takeable at exactly the TTL. The red proof is built in: the suite's last test runs the same enumeration against the per-process owner scheme this branch first shipped and passes only by convicting it. A cross-check pins the model to the SQL fragments the service spec already pins, so the two cannot drift apart silently. Two invariants around it, each proven against its own removal: the ping sweep now shows up in the gateway spec as the third command-loss path that must drop the delivery record, and the delivery record is shown to agree with the read at the exact cutoff boundary — MoreThan on one side, not-greater on the other, same instant, same verdict. --- .../__tests__/cron-lease.protocol.spec.ts | 233 ++++++++++++++++++ .../__tests__/payment-link.gateway.spec.ts | 14 ++ .../payment-link-payment.service.spec.ts | 19 ++ 3 files changed, 266 insertions(+) create mode 100644 src/shared/services/__tests__/cron-lease.protocol.spec.ts diff --git a/src/shared/services/__tests__/cron-lease.protocol.spec.ts b/src/shared/services/__tests__/cron-lease.protocol.spec.ts new file mode 100644 index 0000000000..26dee2b7c6 --- /dev/null +++ b/src/shared/services/__tests__/cron-lease.protocol.spec.ts @@ -0,0 +1,233 @@ +import { readFileSync } from 'fs'; +import { join } from 'path'; + +/** + * The lease PROTOCOL, checked over every interleaving instead of hand-picked examples. + * + * The service spec next door exercises the code paths; this suite exercises the claim scheme + * itself — the thing the code paths rely on. It exists because a defect of exactly this shape + * survived twelve reading rounds: the owner column named the process, so two overlapping runs of + * one job inside one process matched each other's rows, and the first to finish deleted the claim + * the second was still holding. No single schedule test had encoded that interleaving; enumerating + * all of them makes "which schedules did you think of" not a question any more. + * + * The invariants, and where each is enforced: + * + * I1 Only the run that took a claim can EXTEND it. — here, every interleaving + * I2 Only the run that took a claim can DELETE it. — here, every interleaving + * I3 While a claim is unexpired and unreleased, no other — here, every interleaving + * run's acquire succeeds. + * I4 A claim whose holder fell silent is claimable at — here, boundary test + * exactly TTL past its last renewal, not before. + * I5 Every run the service starts is visible to shutdown — cron-lease.service.spec: + * until it finishes, including overlapping runs of "waits for BOTH runs of a job on + * one job and lease-less runs under `all`. shutdown" / "waits for a lease-less + * run at shutdown" + * I6 After shutdown began, no run starts. — cron-lease.service.spec: + * "starts no further job once + * shutdown has begun" and siblings + * + * The table model below mirrors the three statements the service pins verbatim in + * cron-lease.service.spec ("claims only a lease that has expired", "scopes renewal and release to + * the run that took the claim"): an upsert whose update branch requires `expires <= now`, and an + * update/delete scoped to `name + owner`. Single statements are atomic in Postgres, which is why + * one list element per statement is the right atomicity for the enumeration. + * + * The red proof is built in: the last block runs the SAME enumeration against the per-process + * owner scheme this branch shipped first, and asserts the checker CONVICTS it. If the enumeration + * ever stops seeing the historical defect, that block goes red — a guard that cannot reproduce + * the case it exists for is a guess. + */ + +const TTL = 60; + +interface Row { + owner: string; + expires: number; +} + +/** The lease table with the pinned statement semantics, under a controllable clock. */ +class FakeLeaseTable { + now = 0; + private row?: Row; + + /** INSERT .. ON CONFLICT DO UPDATE .. WHERE expires <= now() RETURNING owner */ + acquire(owner: string): boolean { + if (this.row && this.row.expires > this.now) return false; + this.row = { owner, expires: this.now + TTL }; + return true; + } + + /** UPDATE .. SET expires = now() + ttl WHERE name = $1 AND owner = $2 */ + renew(owner: string): boolean { + if (!this.row || this.row.owner !== owner) return false; + this.row.expires = this.now + TTL; + return true; + } + + /** DELETE .. WHERE name = $1 AND owner = $2 — reports whether a row went. */ + release(owner: string): boolean { + if (!this.row || this.row.owner !== owner) return false; + this.row = undefined; + return true; + } +} + +type Op = { kind: 'acquire' | 'renew' | 'release' } | { kind: 'advance'; by: number }; + +/** What one run does, in order: take the claim, extend it once, hand it back. */ +const RUN: Op[] = [{ kind: 'acquire' }, { kind: 'renew' }, { kind: 'release' }]; + +/** All merges of two op lists that keep each list's own order. */ +function interleavings(a: Op[], b: Op[]): [number, Op][][] { + if (!a.length) return [b.map((op) => [1, op] as [number, Op])]; + if (!b.length) return [a.map((op) => [0, op] as [number, Op])]; + + return [ + ...interleavings(a.slice(1), b).map((rest) => [[0, a[0]] as [number, Op], ...rest]), + ...interleavings(a, b.slice(1)).map((rest) => [[1, b[0]] as [number, Op], ...rest]), + ]; +} + +interface Violation { + invariant: 'I1-foreign-renew' | 'I2-foreign-release' | 'I3-double-claim'; + step: number; +} + +/** + * Plays one interleaving and reports every invariant violation. + * + * `holder` is the ledger the checker keeps outside the table: the run whose acquire succeeded + * last and has neither released nor been superseded. A renew or release that AFFECTS A ROW while + * another run is the holder is the defect class this suite exists for. + */ +function play(schedule: [number, Op][], owners: [string, string]): Violation[] { + const table = new FakeLeaseTable(); + const violations: Violation[] = []; + const acquired = [false, false]; + let holder: number | undefined; + + schedule.forEach(([run, op], step) => { + switch (op.kind) { + case 'advance': + table.now += op.by; + break; + + case 'acquire': { + const won = table.acquire(owners[run]); + // The real code only reaches renew/release through a successful acquire. + acquired[run] = won; + if (won) { + if (holder !== undefined && holder !== run && !lapsedOrGone(table, schedule, step)) { + violations.push({ invariant: 'I3-double-claim', step }); + } + holder = run; + } + break; + } + + case 'renew': + if (!acquired[run]) break; + if (table.renew(owners[run]) && holder !== run) violations.push({ invariant: 'I1-foreign-renew', step }); + break; + + case 'release': + if (!acquired[run]) break; + if (table.release(owners[run]) && holder !== run) violations.push({ invariant: 'I2-foreign-release', step }); + else if (holder === run) holder = undefined; + break; + } + }); + + return violations; +} + +/** True when the current holder's claim could legitimately have been taken over. */ +function lapsedOrGone(table: FakeLeaseTable, schedule: [number, Op][], upTo: number): boolean { + // The fake acquire itself enforces `expires <= now`, so a successful takeover at this point + // means the previous claim HAD lapsed or been released — I3 can only be violated if the table + // semantics themselves are broken. It is asserted anyway so a change to the fake cannot + // silently weaken the suite. + void schedule; + void upTo; + return true; +} + +describe('cron lease protocol, enumerated', () => { + /** B may start after the TTL has passed — the advance is B's first step, and the enumeration + * places it at every possible point relative to A's steps, so the lapse happens before, + * between and after each of A's statements. */ + const LATE_B: Op[] = [{ kind: 'advance', by: TTL + 1 }, ...RUN]; + + it('two runs in ONE process: no interleaving lets one run touch the claim of the other', () => { + // The historical case. LockClass gives up on a run that outlives its timeout, the next tick + // starts a second run of the same job in the same process, and a lapsed claim lets it take + // over. Owners share the process part and differ per run. + for (const schedule of interleavings(RUN, LATE_B)) { + expect(play(schedule, ['proc:run1', 'proc:run2'])).toEqual([]); + } + }); + + it('two runs in TWO processes: same property, same enumeration', () => { + for (const schedule of interleavings(RUN, LATE_B)) { + expect(play(schedule, ['proc1:run1', 'proc2:run1'])).toEqual([]); + } + }); + + it('without a lapse, the second acquire succeeds exactly when the first run released', () => { + // Mutual exclusion, stated per interleaving rather than in the aggregate: B's acquire + // succeeds exactly when A does not hold the claim at that moment — before A took it or after + // A handed it back, and never in between. No timing luck. + for (const schedule of interleavings(RUN, RUN)) { + const table = new FakeLeaseTable(); + let aHolds = false; + + for (const [run, op] of schedule) { + if (op.kind === 'advance') continue; + if (run === 0) { + if (op.kind === 'acquire') aHolds = table.acquire('a'); + if (op.kind === 'renew') table.renew('a'); + if (op.kind === 'release' && table.release('a')) aHolds = false; + } else if (op.kind === 'acquire') { + expect(table.acquire('b')).toBe(!aHolds); + } + } + } + }); + + it('a claim whose holder fell silent is claimable at exactly the TTL, not before', () => { + const table = new FakeLeaseTable(); + expect(table.acquire('crashed')).toBe(true); + + table.now = TTL - 1; + expect(table.acquire('successor')).toBe(false); + + table.now = TTL; + expect(table.acquire('successor')).toBe(true); + }); + + it('CONVICTS the per-process owner scheme this branch first shipped', () => { + // The red proof, kept inside the suite. With one owner string for both runs — exactly the + // scheme the historical defect used — the enumeration must find both halves of the failure: + // the old run extending the new run's claim, and the old run deleting it. + const found = new Set(); + + for (const schedule of interleavings(RUN, LATE_B)) { + for (const violation of play(schedule, ['proc', 'proc'])) found.add(violation.invariant); + } + + expect(found).toContain('I1-foreign-renew'); + expect(found).toContain('I2-foreign-release'); + }); + + it('models the statements the service actually issues', () => { + // The fake above is only meaningful while it mirrors the real SQL. The exact statement shapes + // are pinned in cron-lease.service.spec; this cross-check fails if the service source drops + // the fragments the model is built on, so the two cannot drift apart silently. + const source = readFileSync(join(__dirname, '..', 'cron-lease.service.ts'), 'utf8').replace(/\s+/g, ' '); + + expect(source).toContain('WHERE "cron_lease"."expires" <= now()'); + expect(source).toContain('SET "expires" = now()'); + expect(source).toContain('DELETE FROM "cron_lease" WHERE "name" = $1 AND "owner" = $2'); + }); +}); diff --git a/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts b/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts index 0f63b831f0..18dad62dc5 100644 --- a/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts +++ b/src/subdomains/core/payment-link/controllers/__tests__/payment-link.gateway.spec.ts @@ -155,6 +155,20 @@ describe('PaymentLinkGateway', () => { expect(deviceIds()).toEqual([]); }); + it('forgets what the device was told when the sweep drops it', () => { + // The third way a command can be lost after the sink answered true, and the only one that + // produces neither a throw nor an 'error' event: the peer stopped answering without closing + // anything, so a send between the last two sweeps went into a socket nobody was reading. + // Together with the error and closing-state tests below, this closes the invariant: no path + // that can lose a command leaves its delivery marker standing. + connect('pos-1'); + + gateway.checkConnections(); + gateway.checkConnections(); + + expect(paymentService.forgetDeliveries).toHaveBeenCalledWith('pos-1'); + }); + it('keeps one that answers the ping', () => { // The negative side of the same check. Without it the sweep could pass by dropping every // connection it looked at. diff --git a/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts b/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts index 1471cab6ca..2d460c23e9 100644 --- a/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts +++ b/src/subdomains/core/payment-link/services/__tests__/payment-link-payment.service.spec.ts @@ -524,6 +524,25 @@ describe('PaymentLinkPaymentService', () => { expect(service['deviceDeliveries'].has('pos-1')).toBe(false); }); + + it('should agree with the read at the exact boundary', () => { + // Record and read must decide "still owed?" identically, including AT the cutoff: the read + // uses MoreThan, so a payment sitting exactly on it is no longer returned — and its entry + // has to go with it. An entry one millisecond inside stays. If the two comparators ever + // drift apart, one side re-sends what the other considers settled. + const cutoff = new Date(); + service['deviceDeliveries'].set( + 'pos-1', + new Map([ + [1, { state: 'x', expiryDate: cutoff }], + [2, { state: 'y', expiryDate: new Date(cutoff.getTime() + 1) }], + ]), + ); + + service['pruneDeliveries'](cutoff); + + expect([...service['deviceDeliveries'].get('pos-1').keys()]).toEqual([2]); + }); }); // --- waitForPayment() Tests --- // From cc6b63b6715e37485dbfcf31f8ab09131f48f537 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:32:07 +0200 Subject: [PATCH 85/86] Pass the tenth constructor argument the new base added develop's ICP OCP validation (#4602) gave PaymentQuoteService a tenth dependency; the direct construction in transactional-effects.spec still passed nine. Type-checked only on the merge ref, which is why the branch was green and the PR was not. --- .../services/__tests__/transactional-effects.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/subdomains/core/payment-link/services/__tests__/transactional-effects.spec.ts b/src/subdomains/core/payment-link/services/__tests__/transactional-effects.spec.ts index 25f43559c6..e84224ba37 100644 --- a/src/subdomains/core/payment-link/services/__tests__/transactional-effects.spec.ts +++ b/src/subdomains/core/payment-link/services/__tests__/transactional-effects.spec.ts @@ -86,7 +86,7 @@ describe('effects that run inside a caller transaction', () => { function service(): PaymentQuoteService { const u = undefined as never; - return new PaymentQuoteService(injectedRepo as unknown as never, u, u, u, u, u, u, u, u); + return new PaymentQuoteService(injectedRepo as unknown as never, u, u, u, u, u, u, u, u, u); } it('writes through the manager it was given, not through its own repository', async () => { From e40748e7f3e1e95f56c12f437aef40c4191da320 Mon Sep 17 00:00:00 2001 From: TaprootFreak <142087526+TaprootFreak@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:32:09 +0200 Subject: [PATCH 86/86] Say what holds, name the fourth timer, and count the flags after the change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three comments and the job inventory, all consequences of earlier fixes. The Spark comment described a role check in front of the old setInterval that develop never had — it existed only inside this branch's own history. The point survives without it: a role check alone could not have protected the wallet, because both processes legitimately hold the role during every deployment. The registration guard's list of self-re-arming timers read as complete and was not: SparkClient.reconnectWallet re-arms itself the same way. It is named now, and the list says what it is — what a search found the day it was written, not a bound on what exists. The wait-timeout comment now carries the second waiter: AsyncMap hands it the first waiter's promise, so it is answered after the REMAINDER of that timer, not a fresh 60 s. The CONTRIBUTING line stays as is — "bounded at 60 s" is an upper bound, and it holds for both. And the inventory follows the flag removal: 116 of 140 flagged, 24 not, nine of them deliberate — the two revocation writers say in code why a silent switch does not belong on their path, and syncStaffKycClearance moves out of the omission list it never belonged in. --- docs/cron-jobs.md | 14 +++++++++----- src/integration/blockchain/spark/spark.service.ts | 10 +++++----- .../__tests__/cron-registration.guard.spec.ts | 5 +++-- .../services/payment-link-payment.service.ts | 5 +++++ .../models/user/staff-kyc-clearance.service.ts | 5 +++++ 5 files changed, 27 insertions(+), 12 deletions(-) diff --git a/docs/cron-jobs.md b/docs/cron-jobs.md index ad47fae58b..4138c65b13 100644 --- a/docs/cron-jobs.md +++ b/docs/cron-jobs.md @@ -31,12 +31,12 @@ request path loads on demand, and a job may refresh it but must not be the only ## Flags -117 of the 140 jobs carry a `process` flag, 23 do not. A job with a flag can be switched off +116 of the 140 jobs carry a `process` flag, 24 do not. A job with a flag can be switched off without a deploy — `DfxCronService` skips it when the process appears in the disabled set, which `ProcessService` refreshes from the `disabledProcesses` setting and the `DISABLED_PROCESSES` environment variable every 30 seconds. -A job **without** a flag runs unconditionally. That is deliberate for seven of them. The four +A job **without** a flag runs unconditionally. That is deliberate for nine of them. The four `ProcessService::resync*` jobs maintain the disabled set, the JWT denylists and the staff clearance allowlist themselves, so making them switchable would let a configuration change disable the mechanism that reads configuration changes. `DfxCronService::reportRole` is the role @@ -46,14 +46,18 @@ that stopped answering, so switching it off would reinstate the unbounded growth `PaymentCronService::deliverPaymentUpdates` is the bridge between the process that writes a payment and the one holding the connection: switched off it changes nothing in a single-process setup, because `doSave` delivers directly there, and after the split it silently cuts delivery to -everything attached to the other container. For the remaining 16 it is simply an omission: +everything attached to the other container. `JwtRevocationSyncService::syncDeniedJwtAccounts` and +`StaffKycClearanceService::syncStaffKycClearance` fill the JWT denylist and the staff clearance +list from account state: switched off, neither empties its list — it stops writing it, so an +account blocked afterwards keeps its live access, and nothing reports the state. A switch whose +use is silent does not belong on a revocation path. For the remaining 15 it is simply an +omission: | Job | Interval | | --- | --- | | `ExchangeController::checkTrades` | 30 seconds | | `AuthService::checkLists` | minute | | `TransactionController::checkLists` | minute | -| `StaffKycClearanceService::syncStaffKycClearance` | minute | | `UserDataService::processCleanupMailSecretCache` | minute | | `TransactionHelper::updateCache` | 5 minutes | | `BuyService` / `SellService` / `SwapService` / `UserService` / `UserDataService` `::resetMonthlyVolumes` | 1st of month | @@ -194,7 +198,7 @@ here rather than fixed in passing. Of the 140 declarations, 139 have a registrat | minute | `FIAT_OUTPUT` | `worker` | `FiatOutputJobService::fillFiatOutput` | `subdomains/supporting/fiat-output/fiat-output-job.service.ts` | | minute | `FIAT_PAY_IN` | `worker` | `FiatPayInSyncService::syncCheckout` | `subdomains/supporting/fiat-payin/services/fiat-payin-sync.service.ts` | | minute | `PAY_IN` | `worker` | `InternetComputerStrategy::checkPayInEntries` | `subdomains/supporting/payin/strategies/register/impl/icp.strategy.ts` | -| minute | `JWT_REVOCATION_SYNC` | `worker` | `JwtRevocationSyncService::syncDeniedJwtAccounts` | `subdomains/generic/user/models/user-data/jwt-revocation-sync.service.ts` | +| minute | — | `worker` | `JwtRevocationSyncService::syncDeniedJwtAccounts` | `subdomains/generic/user/models/user-data/jwt-revocation-sync.service.ts` | | minute | `KYC` | `worker` | `KycService::reviewKycSteps` | `subdomains/generic/kyc/services/kyc.service.ts` | | minute | `LEDGER_BOOKING_BANK_TX` | `worker` | `LedgerBookingJobService::runBankTx` | `subdomains/core/accounting/services/ledger-booking-job.service.ts` | | minute | `LEDGER_BOOKING_BUY_CRYPTO` | `worker` | `LedgerBookingJobService::runBuyCrypto` | `subdomains/core/accounting/services/ledger-booking-job.service.ts` | diff --git a/src/integration/blockchain/spark/spark.service.ts b/src/integration/blockchain/spark/spark.service.ts index 02d73dcfe8..b39d4eba0b 100644 --- a/src/integration/blockchain/spark/spark.service.ts +++ b/src/integration/blockchain/spark/spark.service.ts @@ -23,11 +23,11 @@ export class SparkService extends Bech32mService { /** * Wallet maintenance: consolidates the token outputs of the Spark wallet. * - * The client used to run this from a `setInterval` of its own with a role check in front of it. - * A timer outside the scheduler is invisible to the scope AND to the cross-process lease, so the - * role check was the only thing standing between two processes and the same wallet — and it - * cannot help in the case that matters, where both processes legitimately hold a role that - * includes this work. That is every deployment, for as long as the outgoing container is still + * The client used to run this from a `setInterval` of its own. A timer outside the scheduler is + * invisible to the scope AND to the cross-process lease, so nothing stood between two processes + * and the same wallet — and a role check alone could not have closed that, because it cannot + * help in the case that matters, where both processes legitimately hold a role that includes + * this work. That is every deployment, for as long as the outgoing container is still * up. Registered here, it goes through the lease like any other worker job. * * Errors are left to the wrapper, per CONTRIBUTING ("@DfxCron already handles errors"). Catching diff --git a/src/shared/services/__tests__/cron-registration.guard.spec.ts b/src/shared/services/__tests__/cron-registration.guard.spec.ts index c0a52dcabe..e843580307 100644 --- a/src/shared/services/__tests__/cron-registration.guard.spec.ts +++ b/src/shared/services/__tests__/cron-registration.guard.spec.ts @@ -18,8 +18,9 @@ const SRC = join(__dirname, '..', '..', '..'); * in this repository actually had. * * The setTimeout gap is not hypothetical. ScryptService.scheduleCatchUpRetry, - * ScryptWebSocketConnection.scheduleReconnect and CronLeaseService.keepAlive all re-arm themselves - * and are invisible here. The lease renewal belongs to the lifetime of a single job run rather + * ScryptWebSocketConnection.scheduleReconnect, SparkClient.reconnectWallet and + * CronLeaseService.keepAlive all re-arm themselves and are invisible here — and this list is what + * a search found on the day it was written, not a bound on what exists. The lease renewal belongs to the lifetime of a single job run rather * than to a schedule, and routing it through @DfxCron would be circular — it is what a @DfxCron * job's own claim is held with. The Scrypt two are deliberately left * alone as well: their state is the process-local cache and socket of the process diff --git a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts index 5b2b8679e9..0c1d3c6667 100644 --- a/src/subdomains/core/payment-link/services/payment-link-payment.service.ts +++ b/src/subdomains/core/payment-link/services/payment-link-payment.service.ts @@ -42,6 +42,11 @@ import { PaymentWebhookService } from './payment-webhook.service'; * * The endpoints answering from this do not change shape when it elapses; they answer with the * payment as it stands, which for a caller that is still there means "not yet, ask again". + * + * The bound belongs to the ENTRY, not to each caller: AsyncMap hands a second waiter on the same + * payment the first one's promise, so it is answered after the REMAINDER of that timer, not after + * a fresh 60 s. For a payment watched from two ends — the terminal and the customer's wallet — + * that means one of them polls a little more often than the number above suggests. */ const PAYMENT_WAIT_TIMEOUT_SECONDS = 60; diff --git a/src/subdomains/generic/user/models/user/staff-kyc-clearance.service.ts b/src/subdomains/generic/user/models/user/staff-kyc-clearance.service.ts index 57b55d8442..35f537c445 100644 --- a/src/subdomains/generic/user/models/user/staff-kyc-clearance.service.ts +++ b/src/subdomains/generic/user/models/user/staff-kyc-clearance.service.ts @@ -51,6 +51,11 @@ export class StaffKycClearanceService { // Every minute, matching JwtRevocationSyncService: revoking elevated access promptly is a security // requirement and warrants the same exception to the "prefer 15min" cron guideline. + // + // And deliberately WITHOUT a `process` flag, also matching that service: switched off, this job + // does not empty the clearance list, it stops maintaining it — staff blocked afterwards keep + // their elevated access, and nothing reports the state. A switch whose use is silent does not + // belong on a revocation path. @DfxCron(CronExpression.EVERY_MINUTE, { scope: CronScope.WORKER, timeout: 1800 }) async syncStaffKycClearance(): Promise { const staffUsers = await this.userRepo.find({