From 1178a3fdfb13a3abb556a293c70720291ec06de3 Mon Sep 17 00:00:00 2001 From: Daniel Padrino Date: Tue, 21 Jul 2026 10:18:45 -0300 Subject: [PATCH 1/2] fix(rpc): stop multicall cache from replaying transient errors forever ethers-multicall-provider's internal DataLoader (cache enabled by default) caches per-call Error results and only clears entries when a load fulfills. A single transient RPC failure (e.g. one HTTP 500) on a multicall batch is therefore replayed from the cache for every future identical call (same to+data+blockTag): retries and later monitoring cycles never reach the RPC again and the service stays down until a process restart. Wrap with cache=false: per-tick call batching is unaffected, and since latest-block entries were cleared after every successful load anyway, no effective caching is lost. withRetry becomes able to actually recover from a transient multicall failure. --- package.json | 3 + src/monitoringV2/provider.service.spec.ts | 76 +++++++++++++++++++++++ src/monitoringV2/provider.service.ts | 6 +- 3 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 src/monitoringV2/provider.service.spec.ts diff --git a/package.json b/package.json index dc32332..9f0eaf6 100644 --- a/package.json +++ b/package.json @@ -77,6 +77,9 @@ ], "rootDir": "src", "testRegex": ".*\\.spec\\.ts$", + "moduleNameMapper": { + "^src/(.*)$": "/$1" + }, "transform": { "^.+\\.(t|j)s$": "ts-jest" }, diff --git a/src/monitoringV2/provider.service.spec.ts b/src/monitoringV2/provider.service.spec.ts new file mode 100644 index 0000000..b92d819 --- /dev/null +++ b/src/monitoringV2/provider.service.spec.ts @@ -0,0 +1,76 @@ +import { ethers } from 'ethers'; +import { ProviderService } from './provider.service'; +import { AppConfigService } from '../config/config.service'; + +const OWNER = '0x01AE4C18c2677f97BaB536C48d6C36858f5c86D7'; +const POSITION = '0x489c40401d465a632297c5810b0e209059e71be4'; + +// Regression test for the 2026-07-21 monitoring outage: ethers-multicall-provider's internal +// DataLoader caches per-call Error results and only clears entries when a load FULFILLS, so with +// caching enabled a single transient RPC failure (e.g. one HTTP 500) is replayed from the cache +// for every future identical call (same to+data+blockTag) — retries never reach the RPC again and +// the service can only recover through a process restart. +describe('ProviderService multicall error handling', () => { + let service: ProviderService; + let ethCallCount: number; + let failNextEthCall: boolean; + + const coder = ethers.AbiCoder.defaultAbiCoder(); + + // tryAggregate(false, [(to, callData)]) -> [(success, returnData)] with owner() succeeding + const tryAggregateResult = () => { + const ownerBytes = coder.encode(['address'], [OWNER]); + return coder.encode(['tuple(bool,bytes)[]'], [[[true, ownerBytes]]]); + }; + + const fakeSend = async (payload: any): Promise => { + const requests = Array.isArray(payload) ? payload : [payload]; + if (failNextEthCall && requests.some((r) => r.method === 'eth_call')) { + failNextEthCall = false; + throw ethers.makeError('server response 500 Internal Server Error', 'SERVER_ERROR', { + request: {} as any, + response: {} as any, + }); + } + return requests.map((r) => { + if (r.method === 'eth_chainId') return { id: r.id, jsonrpc: '2.0', result: '0x1' }; + if (r.method === 'eth_call') { + ethCallCount++; + return { id: r.id, jsonrpc: '2.0', result: tryAggregateResult() }; + } + throw new Error(`unexpected method ${r.method}`); + }); + }; + + beforeEach(() => { + ethCallCount = 0; + failNextEthCall = true; + const config = { rpcUrl: 'http://127.0.0.1:1', rpcTimeoutMs: 1000 } as AppConfigService; + service = new ProviderService(config); + (service.provider as any)._send = fakeSend; + }); + + afterEach(() => { + (service.provider as any).destroy?.(); + }); + + const ownerCall = () => { + const position = new ethers.Contract(POSITION, ['function owner() view returns (address)'], service.multicallProvider); + return position.owner() as Promise; + }; + + it('does not replay a transient multicall failure from the cache on the next identical call', async () => { + await expect(ownerCall()).rejects.toThrow('server response 500'); + + // The RPC is healthy again; the identical call must be re-sent, not served the cached error. + await expect(ownerCall()).resolves.toBe(OWNER); + expect(ethCallCount).toBe(1); + }); + + it('recovers from a transient multicall failure within a single withRetry loop', async () => { + // Mirrors the production shape: callBatch/call wrap the same thunk in withRetry, so the retry + // only helps if the second attempt actually reaches the RPC. + await expect(service.call(ownerCall, 3)).resolves.toBe(OWNER); + expect(ethCallCount).toBe(1); + }); +}); diff --git a/src/monitoringV2/provider.service.ts b/src/monitoringV2/provider.service.ts index e5cd3dd..5651971 100644 --- a/src/monitoringV2/provider.service.ts +++ b/src/monitoringV2/provider.service.ts @@ -115,7 +115,11 @@ export class ProviderService { this.logger.log( `LoggingJsonRpcProvider initialized with _send() override for RPC call tracking (timeout: ${this.config.rpcTimeoutMs}ms)` ); - this.multicallProviderInstance = MulticallWrapper.wrap(this.ethersProvider, ProviderService.CALLEDATA_LIMIT); + // cache=false: the wrapper's DataLoader caches per-call Error results and only clears entries on + // success, so a single transient RPC failure would otherwise be replayed from the cache for every + // future identical call (same to+data+blockTag) without ever hitting the RPC again — retries + // included. Latest-block results were cleared after each load anyway, so no caching is lost. + this.multicallProviderInstance = MulticallWrapper.wrap(this.ethersProvider, ProviderService.CALLEDATA_LIMIT, false); this.logger.log( `Multicall provider initialized with ${ProviderService.CALLEDATA_LIMIT} bytes calldata limit and ${this.config.rpcTimeoutMs}ms timeout` ); From 584204426b530527dd61fad0d2869b55f58f25c7 Mon Sep 17 00:00:00 2001 From: Daniel Padrino Date: Tue, 21 Jul 2026 10:29:40 -0300 Subject: [PATCH 2/2] test(rpc): finish provider network bootstrap before arming the failing batch Otherwise the intentionally failed first batch also takes down the lazy eth_chainId network detection, which logs noise and leaves a 1s retry timer that can trigger jest's force-exit warning. --- src/monitoringV2/provider.service.spec.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/monitoringV2/provider.service.spec.ts b/src/monitoringV2/provider.service.spec.ts index b92d819..950eee9 100644 --- a/src/monitoringV2/provider.service.spec.ts +++ b/src/monitoringV2/provider.service.spec.ts @@ -42,12 +42,17 @@ describe('ProviderService multicall error handling', () => { }); }; - beforeEach(() => { + beforeEach(async () => { ethCallCount = 0; - failNextEthCall = true; + failNextEthCall = false; const config = { rpcUrl: 'http://127.0.0.1:1', rpcTimeoutMs: 1000 } as AppConfigService; service = new ProviderService(config); (service.provider as any)._send = fakeSend; + // Complete the provider's lazy network bootstrap against the healthy fake before arming the + // failure, so the intentionally failed batch cannot drag eth_chainId down with it (that would + // leave a 1s network-detection retry timer behind and log noise in every run). + await service.provider.send('eth_chainId', []); + failNextEthCall = true; }); afterEach(() => {