fix(rpc): stop multicall cache from replaying transient errors forever

package.json

@@ -77,6 +77,9 @@
     ],
     "rootDir": "src",
     "testRegex": ".*\\.spec\\.ts$",
+    "moduleNameMapper": {
+      "^src/(.*)$": "<rootDir>/$1"
+    },
     "transform": {
       "^.+\\.(t|j)s$": "ts-jest"
     },

src/monitoringV2/provider.service.spec.ts

@@ -0,0 +1,81 @@
+import { ethers } from 'ethers';
+import { ProviderService } from './provider.service';
+import { AppConfigService } from '../config/config.service';
+
+const OWNER = '0x01AE4C18c2677f97BaB536C48d6C36858f5c86D7';
+const POSITION = '0x489c40401d465a632297c5810b0e209059e71be4';
+
+// Regression test for the 2026-07-21 monitoring outage: ethers-multicall-provider's internal
+// DataLoader caches per-call Error results and only clears entries when a load FULFILLS, so with
+// caching enabled a single transient RPC failure (e.g. one HTTP 500) is replayed from the cache
+// for every future identical call (same to+data+blockTag) — retries never reach the RPC again and
+// the service can only recover through a process restart.
+describe('ProviderService multicall error handling', () => {
+	let service: ProviderService;
+	let ethCallCount: number;
+	let failNextEthCall: boolean;
+
+	const coder = ethers.AbiCoder.defaultAbiCoder();
+
+	// tryAggregate(false, [(to, callData)]) -> [(success, returnData)] with owner() succeeding
+	const tryAggregateResult = () => {
+		const ownerBytes = coder.encode(['address'], [OWNER]);
+		return coder.encode(['tuple(bool,bytes)[]'], [[[true, ownerBytes]]]);
+	};
+
+	const fakeSend = async (payload: any): Promise<any> => {
+		const requests = Array.isArray(payload) ? payload : [payload];
+		if (failNextEthCall && requests.some((r) => r.method === 'eth_call')) {
+			failNextEthCall = false;
+			throw ethers.makeError('server response 500 Internal Server Error', 'SERVER_ERROR', {
+				request: {} as any,
+				response: {} as any,
+			});
+		}
+		return requests.map((r) => {
+			if (r.method === 'eth_chainId') return { id: r.id, jsonrpc: '2.0', result: '0x1' };
+			if (r.method === 'eth_call') {
+				ethCallCount++;
+				return { id: r.id, jsonrpc: '2.0', result: tryAggregateResult() };
+			}
+			throw new Error(`unexpected method ${r.method}`);
+		});
+	};
+
+	beforeEach(async () => {
+		ethCallCount = 0;
+		failNextEthCall = false;
+		const config = { rpcUrl: 'http://127.0.0.1:1', rpcTimeoutMs: 1000 } as AppConfigService;
+		service = new ProviderService(config);
+		(service.provider as any)._send = fakeSend;
+		// Complete the provider's lazy network bootstrap against the healthy fake before arming the
+		// failure, so the intentionally failed batch cannot drag eth_chainId down with it (that would
+		// leave a 1s network-detection retry timer behind and log noise in every run).
+		await service.provider.send('eth_chainId', []);
+		failNextEthCall = true;
+	});
+
+	afterEach(() => {
+		(service.provider as any).destroy?.();
+	});
+
+	const ownerCall = () => {
+		const position = new ethers.Contract(POSITION, ['function owner() view returns (address)'], service.multicallProvider);
+		return position.owner() as Promise<string>;
+	};
+
+	it('does not replay a transient multicall failure from the cache on the next identical call', async () => {
+		await expect(ownerCall()).rejects.toThrow('server response 500');
+
+		// The RPC is healthy again; the identical call must be re-sent, not served the cached error.
+		await expect(ownerCall()).resolves.toBe(OWNER);
+		expect(ethCallCount).toBe(1);
+	});
+
+	it('recovers from a transient multicall failure within a single withRetry loop', async () => {
+		// Mirrors the production shape: callBatch/call wrap the same thunk in withRetry, so the retry
+		// only helps if the second attempt actually reaches the RPC.
+		await expect(service.call(ownerCall, 3)).resolves.toBe(OWNER);
+		expect(ethCallCount).toBe(1);
+	});
+});

src/monitoringV2/provider.service.ts

@@ -115,7 +115,11 @@ export class ProviderService {
 		this.logger.log(
 			`LoggingJsonRpcProvider initialized with _send() override for RPC call tracking (timeout: ${this.config.rpcTimeoutMs}ms)`
 		);
-		this.multicallProviderInstance = MulticallWrapper.wrap(this.ethersProvider, ProviderService.CALLEDATA_LIMIT);
+		// cache=false: the wrapper's DataLoader caches per-call Error results and only clears entries on
+		// success, so a single transient RPC failure would otherwise be replayed from the cache for every
+		// future identical call (same to+data+blockTag) without ever hitting the RPC again — retries
+		// included. Latest-block results were cleared after each load anyway, so no caching is lost.
+		this.multicallProviderInstance = MulticallWrapper.wrap(this.ethersProvider, ProviderService.CALLEDATA_LIMIT, false);
 		this.logger.log(
 			`Multicall provider initialized with ${ProviderService.CALLEDATA_LIMIT} bytes calldata limit and ${this.config.rpcTimeoutMs}ms timeout`
 		);