From fa2773fa765fb7fd4be780953df3d34ff2864cda Mon Sep 17 00:00:00 2001 From: Pablo Maldonado Date: Fri, 19 Dec 2025 15:24:37 +0000 Subject: [PATCH 1/7] feat(bot-oo): detect nonce backlog and attempt self-tx cancellation --- packages/monitor-v2/src/bot-oo/index.ts | 134 +++++++++++++++++++++++- 1 file changed, 133 insertions(+), 1 deletion(-) diff --git a/packages/monitor-v2/src/bot-oo/index.ts b/packages/monitor-v2/src/bot-oo/index.ts index 1906b64914..965a757ca7 100644 --- a/packages/monitor-v2/src/bot-oo/index.ts +++ b/packages/monitor-v2/src/bot-oo/index.ts @@ -1,8 +1,132 @@ import { delay, waitForLogger, GasEstimator } from "@uma/financial-templates-lib"; -import { BotModes, initMonitoringParams, Logger, startupLogLevel } from "./common"; +import { BigNumber } from "ethers"; +import { BotModes, MonitoringParams, initMonitoringParams, Logger, startupLogLevel } from "./common"; import { settleRequests } from "./SettleRequests"; const logger = Logger; +const DEFAULT_REPLACEMENT_BUMP_NUMERATOR = 12; +const DEFAULT_REPLACEMENT_BUMP_DENOMINATOR = 10; +const DEFAULT_REPLACEMENT_ATTEMPTS = 3; + +type NonceBacklogConfig = { + replacementBumpNumerator: number; + replacementBumpDenominator: number; + replacementAttempts: number; +}; + +const parsePositiveInt = (value: string | undefined, defaultValue: number): number => { + const parsed = Number(value); + return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : defaultValue; +}; + +const getNonceBacklogConfig = (env: NodeJS.ProcessEnv): NonceBacklogConfig => ({ + replacementBumpNumerator: parsePositiveInt(env.NONCE_REPLACEMENT_BUMP_NUMERATOR, DEFAULT_REPLACEMENT_BUMP_NUMERATOR), + replacementBumpDenominator: parsePositiveInt( + env.NONCE_REPLACEMENT_BUMP_DENOMINATOR, + DEFAULT_REPLACEMENT_BUMP_DENOMINATOR + ), + replacementAttempts: parsePositiveInt(env.NONCE_REPLACEMENT_ATTEMPTS, DEFAULT_REPLACEMENT_ATTEMPTS), +}); + +function bumpFeeData( + feeData: ReturnType, + bumps: number, + config: NonceBacklogConfig +): ReturnType { + if (bumps === 0) return feeData; + + const bumpValue = (value: BigNumber) => { + let bumped = value; + for (let i = 0; i < bumps; i++) { + bumped = bumped.mul(config.replacementBumpNumerator).div(config.replacementBumpDenominator); + } + return bumped; + }; + + if ("gasPrice" in feeData) { + return { gasPrice: bumpValue(feeData.gasPrice) }; + } + + return { + maxFeePerGas: bumpValue(feeData.maxFeePerGas), + maxPriorityFeePerGas: bumpValue(feeData.maxPriorityFeePerGas), + }; +} + +async function handleNonceBacklog( + params: MonitoringParams, + gasEstimator: GasEstimator, + config: NonceBacklogConfig +): Promise { + const botAddress = await params.signer.getAddress(); + const [latestNonce, pendingNonce] = await Promise.all([ + params.provider.getTransactionCount(botAddress, "latest"), + params.provider.getTransactionCount(botAddress, "pending"), + ]); + + if (pendingNonce <= latestNonce) return false; + + logger.warn({ + at: "OracleBot", + message: "Nonce backlog detected, skipping settlements for this run", + botAddress, + latestNonce, + pendingNonce, + }); + + await gasEstimator.update(); + const baseFeeData = gasEstimator.getCurrentFastPriceEthers(); + + for (let attempt = 1; attempt <= config.replacementAttempts; attempt++) { + const feeData = bumpFeeData(baseFeeData, attempt - 1, config); + try { + const tx = await params.signer.sendTransaction({ + to: botAddress, + value: 0, + nonce: latestNonce, + gasLimit: 21_000, + ...feeData, + }); + + logger.info({ + at: "OracleBot", + message: "Submitted nonce backlog cancellation transaction", + tx: tx.hash, + nonce: latestNonce, + attempt, + feeData, + }); + + await tx.wait(1); + + logger.info({ + at: "OracleBot", + message: "Nonce backlog cancellation transaction mined", + tx: tx.hash, + nonce: latestNonce, + attempt, + }); + return true; + } catch (error) { + logger.warn({ + at: "OracleBot", + message: "Nonce backlog cancellation transaction failed", + attempt, + nonce: latestNonce, + error, + }); + } + } + + logger.error({ + at: "OracleBot", + message: "Failed to clear nonce backlog, exiting early", + nonce: latestNonce, + pendingNonce, + }); + + return true; +} async function main() { const params = await initMonitoringParams(process.env); @@ -16,12 +140,20 @@ async function main() { }); const gasEstimator = new GasEstimator(logger, undefined, params.chainId, params.provider); + const nonceBacklogConfig = getNonceBacklogConfig(process.env); const cmds = { settleRequestsEnabled: settleRequests, }; for (;;) { + const backlogDetected = await handleNonceBacklog(params, gasEstimator, nonceBacklogConfig); + if (backlogDetected) { + await delay(5); // Let any in-flight logs flush before exiting. + await waitForLogger(logger); + break; + } + await gasEstimator.update(); const runCmds = Object.entries(cmds) From e448f447af008e32829ec10b59ed94e88386aff2 Mon Sep 17 00:00:00 2001 From: Pablo Maldonado Date: Mon, 19 Jan 2026 12:37:33 +0000 Subject: [PATCH 2/7] feat: standalone script Signed-off-by: Pablo Maldonado --- .../transaction-clearer/TransactionClearer.ts | 171 ++++++++++++++++++ .../src/transaction-clearer/common.ts | 34 ++++ .../src/transaction-clearer/index.ts | 56 ++++++ 3 files changed, 261 insertions(+) create mode 100644 packages/monitor-v2/src/transaction-clearer/TransactionClearer.ts create mode 100644 packages/monitor-v2/src/transaction-clearer/common.ts create mode 100644 packages/monitor-v2/src/transaction-clearer/index.ts diff --git a/packages/monitor-v2/src/transaction-clearer/TransactionClearer.ts b/packages/monitor-v2/src/transaction-clearer/TransactionClearer.ts new file mode 100644 index 0000000000..e205162a15 --- /dev/null +++ b/packages/monitor-v2/src/transaction-clearer/TransactionClearer.ts @@ -0,0 +1,171 @@ +import { BigNumber } from "ethers"; +import type { Logger as LoggerType } from "winston"; +import type { Provider } from "@ethersproject/abstract-provider"; +import { GasEstimator } from "@uma/financial-templates-lib"; +import { MonitoringParams, NonceBacklogConfig } from "./common"; + +type FeeData = { maxFeePerGas: BigNumber; maxPriorityFeePerGas: BigNumber } | { gasPrice: BigNumber }; + +function isLondonFeeData(feeData: FeeData): feeData is { maxFeePerGas: BigNumber; maxPriorityFeePerGas: BigNumber } { + return "maxFeePerGas" in feeData; +} + +function bumpFeeData(baseFeeData: FeeData, attemptIndex: number, config: NonceBacklogConfig): FeeData { + // Calculate multiplier: (numerator/denominator)^(attemptIndex+1) + // For attempt 0: 1.2x, attempt 1: 1.44x, attempt 2: 1.73x (with default 12/10) + let numerator = BigNumber.from(config.feeBumpNumerator); + let denominator = BigNumber.from(config.feeBumpDenominator); + + for (let i = 0; i < attemptIndex; i++) { + numerator = numerator.mul(config.feeBumpNumerator); + denominator = denominator.mul(config.feeBumpDenominator); + } + + if (isLondonFeeData(baseFeeData)) { + return { + maxFeePerGas: baseFeeData.maxFeePerGas.mul(numerator).div(denominator), + maxPriorityFeePerGas: baseFeeData.maxPriorityFeePerGas.mul(numerator).div(denominator), + }; + } else { + return { + gasPrice: baseFeeData.gasPrice.mul(numerator).div(denominator), + }; + } +} + +async function getNonces( + provider: Provider, + address: string +): Promise<{ latestNonce: number; pendingNonce: number }> { + const [latestNonce, pendingNonce] = await Promise.all([ + provider.getTransactionCount(address, "latest"), + provider.getTransactionCount(address, "pending"), + ]); + return { latestNonce, pendingNonce }; +} + +export async function clearStuckTransactions( + logger: LoggerType, + params: MonitoringParams, + gasEstimator: GasEstimator +): Promise { + const { provider, signer, nonceBacklogConfig } = params; + const botAddress = await signer.getAddress(); + + const { latestNonce, pendingNonce } = await getNonces(provider, botAddress); + const backlog = pendingNonce - latestNonce; + + if (backlog < nonceBacklogConfig.nonceBacklogThreshold) { + logger.debug({ + at: "TransactionClearer", + message: "No nonce backlog detected", + botAddress, + latestNonce, + pendingNonce, + backlog, + threshold: nonceBacklogConfig.nonceBacklogThreshold, + }); + return; + } + + logger.warn({ + at: "TransactionClearer", + message: "Nonce backlog detected, attempting to clear stuck transactions", + botAddress, + latestNonce, + pendingNonce, + backlog, + threshold: nonceBacklogConfig.nonceBacklogThreshold, + }); + + // Get base fee data from gas estimator + const baseFeeData = gasEstimator.getCurrentFastPriceEthers(); + + // Clear all stuck nonces from latestNonce to pendingNonce - 1 + for (let nonce = latestNonce; nonce < pendingNonce; nonce++) { + let cleared = false; + + for (let attempt = 0; attempt < nonceBacklogConfig.replacementAttempts; attempt++) { + const feeData = bumpFeeData(baseFeeData, attempt, nonceBacklogConfig); + + try { + logger.info({ + at: "TransactionClearer", + message: `Attempting to clear stuck transaction (nonce ${nonce}, attempt ${attempt + 1})`, + botAddress, + nonce, + attempt: attempt + 1, + feeData: isLondonFeeData(feeData) + ? { + maxFeePerGas: feeData.maxFeePerGas.toString(), + maxPriorityFeePerGas: feeData.maxPriorityFeePerGas.toString(), + } + : { gasPrice: feeData.gasPrice.toString() }, + }); + + const tx = await signer.sendTransaction({ + to: botAddress, // Self-transaction + value: 0, + nonce, + gasLimit: 21_000, + ...feeData, + }); + + const receipt = await tx.wait(1); + + logger.info({ + at: "TransactionClearer", + message: `Successfully cleared stuck transaction (nonce ${nonce})`, + botAddress, + nonce, + transactionHash: receipt.transactionHash, + gasUsed: receipt.gasUsed.toString(), + }); + + cleared = true; + break; + } catch (error) { + logger.warn({ + at: "TransactionClearer", + message: `Failed to clear stuck transaction (nonce ${nonce}, attempt ${attempt + 1})`, + botAddress, + nonce, + attempt: attempt + 1, + error: error instanceof Error ? error.message : String(error), + }); + } + } + + if (!cleared) { + logger.error({ + at: "TransactionClearer", + message: `Failed to clear stuck transaction after all attempts (nonce ${nonce})`, + botAddress, + nonce, + maxAttempts: nonceBacklogConfig.replacementAttempts, + }); + } + } + + // Verify final state + const { latestNonce: finalLatestNonce, pendingNonce: finalPendingNonce } = await getNonces(provider, botAddress); + const finalBacklog = finalPendingNonce - finalLatestNonce; + + if (finalBacklog < nonceBacklogConfig.nonceBacklogThreshold) { + logger.info({ + at: "TransactionClearer", + message: "Successfully cleared nonce backlog", + botAddress, + previousBacklog: backlog, + finalBacklog, + }); + } else { + logger.warn({ + at: "TransactionClearer", + message: "Nonce backlog still present after clearing attempt", + botAddress, + previousBacklog: backlog, + finalBacklog, + }); + } +} diff --git a/packages/monitor-v2/src/transaction-clearer/common.ts b/packages/monitor-v2/src/transaction-clearer/common.ts new file mode 100644 index 0000000000..3815de8841 --- /dev/null +++ b/packages/monitor-v2/src/transaction-clearer/common.ts @@ -0,0 +1,34 @@ +export { Logger } from "@uma/financial-templates-lib"; +import { BaseMonitoringParams, initBaseMonitoringParams, startupLogLevel as baseStartup } from "../bot-utils/base"; + +export interface NonceBacklogConfig { + // Minimum nonce difference (pending - latest) to trigger clearing + nonceBacklogThreshold: number; + // Fee bump multiplier: bumpedFee = fee * numerator / denominator + feeBumpNumerator: number; + feeBumpDenominator: number; + // Max attempts to replace a stuck transaction with increasing fees + replacementAttempts: number; +} + +export interface MonitoringParams extends BaseMonitoringParams { + nonceBacklogConfig: NonceBacklogConfig; +} + +export const initMonitoringParams = async (env: NodeJS.ProcessEnv): Promise => { + const base = await initBaseMonitoringParams(env); + + const nonceBacklogConfig: NonceBacklogConfig = { + nonceBacklogThreshold: Number(env.NONCE_BACKLOG_THRESHOLD) || 1, + feeBumpNumerator: Number(env.FEE_BUMP_NUMERATOR) || 12, + feeBumpDenominator: Number(env.FEE_BUMP_DENOMINATOR) || 10, + replacementAttempts: Number(env.REPLACEMENT_ATTEMPTS) || 3, + }; + + return { + ...base, + nonceBacklogConfig, + }; +}; + +export const startupLogLevel = baseStartup; diff --git a/packages/monitor-v2/src/transaction-clearer/index.ts b/packages/monitor-v2/src/transaction-clearer/index.ts new file mode 100644 index 0000000000..7214eb3e2c --- /dev/null +++ b/packages/monitor-v2/src/transaction-clearer/index.ts @@ -0,0 +1,56 @@ +import { delay, waitForLogger, GasEstimator } from "@uma/financial-templates-lib"; +import { initMonitoringParams, Logger, startupLogLevel } from "./common"; +import { clearStuckTransactions } from "./TransactionClearer"; + +const logger = Logger; + +async function main() { + const params = await initMonitoringParams(process.env); + + logger[startupLogLevel(params)]({ + at: "TransactionClearer", + message: "Transaction Clearer Bot started", + chainId: params.chainId, + nonceBacklogConfig: params.nonceBacklogConfig, + }); + + const gasEstimator = new GasEstimator(logger, undefined, params.chainId, params.provider); + + for (;;) { + await gasEstimator.update(); + + try { + await clearStuckTransactions(logger, params, gasEstimator); + } catch (error) { + logger.error({ + at: "TransactionClearer", + message: "Error clearing stuck transactions", + error, + }); + } + + if (params.pollingDelay !== 0) { + await delay(params.pollingDelay); + } else { + await delay(5); // Allow transports to flush + await waitForLogger(logger); + break; + } + } +} + +main().then( + () => { + process.exit(0); + }, + async (error) => { + logger.error({ + at: "TransactionClearer", + message: "Transaction Clearer Bot execution error", + error, + }); + await delay(5); + await waitForLogger(logger); + process.exit(1); + } +); From e7077ccd9dead408386d9a02a67e986cd3eb6b3f Mon Sep 17 00:00:00 2001 From: Pablo Maldonado Date: Mon, 19 Jan 2026 12:57:42 +0000 Subject: [PATCH 3/7] fix: review --- packages/monitor-v2/src/bot-oo/index.ts | 30 +++++++++++++++---------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/packages/monitor-v2/src/bot-oo/index.ts b/packages/monitor-v2/src/bot-oo/index.ts index 965a757ca7..c0994726f3 100644 --- a/packages/monitor-v2/src/bot-oo/index.ts +++ b/packages/monitor-v2/src/bot-oo/index.ts @@ -4,28 +4,34 @@ import { BotModes, MonitoringParams, initMonitoringParams, Logger, startupLogLev import { settleRequests } from "./SettleRequests"; const logger = Logger; -const DEFAULT_REPLACEMENT_BUMP_NUMERATOR = 12; -const DEFAULT_REPLACEMENT_BUMP_DENOMINATOR = 10; +const DEFAULT_REPLACEMENT_BUMP_PERCENT = 20; const DEFAULT_REPLACEMENT_ATTEMPTS = 3; type NonceBacklogConfig = { - replacementBumpNumerator: number; - replacementBumpDenominator: number; + replacementBumpPercent: number; replacementAttempts: number; }; -const parsePositiveInt = (value: string | undefined, defaultValue: number): number => { +const parsePositiveInt = (value: string | undefined, defaultValue: number, name: string): number => { + if (value === undefined) return defaultValue; const parsed = Number(value); - return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : defaultValue; + if (!Number.isFinite(parsed) || parsed <= 0 || !Number.isInteger(parsed)) { + throw new Error(`${name} must be a positive integer, got: ${value}`); + } + return parsed; }; const getNonceBacklogConfig = (env: NodeJS.ProcessEnv): NonceBacklogConfig => ({ - replacementBumpNumerator: parsePositiveInt(env.NONCE_REPLACEMENT_BUMP_NUMERATOR, DEFAULT_REPLACEMENT_BUMP_NUMERATOR), - replacementBumpDenominator: parsePositiveInt( - env.NONCE_REPLACEMENT_BUMP_DENOMINATOR, - DEFAULT_REPLACEMENT_BUMP_DENOMINATOR + replacementBumpPercent: parsePositiveInt( + env.NONCE_REPLACEMENT_BUMP_PERCENT, + DEFAULT_REPLACEMENT_BUMP_PERCENT, + "NONCE_REPLACEMENT_BUMP_PERCENT" + ), + replacementAttempts: parsePositiveInt( + env.NONCE_REPLACEMENT_ATTEMPTS, + DEFAULT_REPLACEMENT_ATTEMPTS, + "NONCE_REPLACEMENT_ATTEMPTS" ), - replacementAttempts: parsePositiveInt(env.NONCE_REPLACEMENT_ATTEMPTS, DEFAULT_REPLACEMENT_ATTEMPTS), }); function bumpFeeData( @@ -38,7 +44,7 @@ function bumpFeeData( const bumpValue = (value: BigNumber) => { let bumped = value; for (let i = 0; i < bumps; i++) { - bumped = bumped.mul(config.replacementBumpNumerator).div(config.replacementBumpDenominator); + bumped = bumped.mul(100 + config.replacementBumpPercent).div(100); } return bumped; }; From 1e03e77f29948d0a6304ddfe1ea478d9f726ec29 Mon Sep 17 00:00:00 2001 From: Pablo Maldonado Date: Mon, 19 Jan 2026 13:14:38 +0000 Subject: [PATCH 4/7] refactor: simplify nonce backlog config and add strict env parsing - Revert bot-oo/index.ts to master (nonce handling moved to transaction-clearer) - Replace feeBumpNumerator/Denominator with simpler feeBumpPercent (default 20%) - Add parsePositiveInt that throws on invalid values instead of silently using defaults Co-Authored-By: Claude Opus 4.5 --- packages/monitor-v2/src/bot-oo/index.ts | 140 +----------------- .../transaction-clearer/TransactionClearer.ts | 24 +-- .../src/transaction-clearer/common.ts | 21 ++- 3 files changed, 27 insertions(+), 158 deletions(-) diff --git a/packages/monitor-v2/src/bot-oo/index.ts b/packages/monitor-v2/src/bot-oo/index.ts index c0994726f3..1906b64914 100644 --- a/packages/monitor-v2/src/bot-oo/index.ts +++ b/packages/monitor-v2/src/bot-oo/index.ts @@ -1,138 +1,8 @@ import { delay, waitForLogger, GasEstimator } from "@uma/financial-templates-lib"; -import { BigNumber } from "ethers"; -import { BotModes, MonitoringParams, initMonitoringParams, Logger, startupLogLevel } from "./common"; +import { BotModes, initMonitoringParams, Logger, startupLogLevel } from "./common"; import { settleRequests } from "./SettleRequests"; const logger = Logger; -const DEFAULT_REPLACEMENT_BUMP_PERCENT = 20; -const DEFAULT_REPLACEMENT_ATTEMPTS = 3; - -type NonceBacklogConfig = { - replacementBumpPercent: number; - replacementAttempts: number; -}; - -const parsePositiveInt = (value: string | undefined, defaultValue: number, name: string): number => { - if (value === undefined) return defaultValue; - const parsed = Number(value); - if (!Number.isFinite(parsed) || parsed <= 0 || !Number.isInteger(parsed)) { - throw new Error(`${name} must be a positive integer, got: ${value}`); - } - return parsed; -}; - -const getNonceBacklogConfig = (env: NodeJS.ProcessEnv): NonceBacklogConfig => ({ - replacementBumpPercent: parsePositiveInt( - env.NONCE_REPLACEMENT_BUMP_PERCENT, - DEFAULT_REPLACEMENT_BUMP_PERCENT, - "NONCE_REPLACEMENT_BUMP_PERCENT" - ), - replacementAttempts: parsePositiveInt( - env.NONCE_REPLACEMENT_ATTEMPTS, - DEFAULT_REPLACEMENT_ATTEMPTS, - "NONCE_REPLACEMENT_ATTEMPTS" - ), -}); - -function bumpFeeData( - feeData: ReturnType, - bumps: number, - config: NonceBacklogConfig -): ReturnType { - if (bumps === 0) return feeData; - - const bumpValue = (value: BigNumber) => { - let bumped = value; - for (let i = 0; i < bumps; i++) { - bumped = bumped.mul(100 + config.replacementBumpPercent).div(100); - } - return bumped; - }; - - if ("gasPrice" in feeData) { - return { gasPrice: bumpValue(feeData.gasPrice) }; - } - - return { - maxFeePerGas: bumpValue(feeData.maxFeePerGas), - maxPriorityFeePerGas: bumpValue(feeData.maxPriorityFeePerGas), - }; -} - -async function handleNonceBacklog( - params: MonitoringParams, - gasEstimator: GasEstimator, - config: NonceBacklogConfig -): Promise { - const botAddress = await params.signer.getAddress(); - const [latestNonce, pendingNonce] = await Promise.all([ - params.provider.getTransactionCount(botAddress, "latest"), - params.provider.getTransactionCount(botAddress, "pending"), - ]); - - if (pendingNonce <= latestNonce) return false; - - logger.warn({ - at: "OracleBot", - message: "Nonce backlog detected, skipping settlements for this run", - botAddress, - latestNonce, - pendingNonce, - }); - - await gasEstimator.update(); - const baseFeeData = gasEstimator.getCurrentFastPriceEthers(); - - for (let attempt = 1; attempt <= config.replacementAttempts; attempt++) { - const feeData = bumpFeeData(baseFeeData, attempt - 1, config); - try { - const tx = await params.signer.sendTransaction({ - to: botAddress, - value: 0, - nonce: latestNonce, - gasLimit: 21_000, - ...feeData, - }); - - logger.info({ - at: "OracleBot", - message: "Submitted nonce backlog cancellation transaction", - tx: tx.hash, - nonce: latestNonce, - attempt, - feeData, - }); - - await tx.wait(1); - - logger.info({ - at: "OracleBot", - message: "Nonce backlog cancellation transaction mined", - tx: tx.hash, - nonce: latestNonce, - attempt, - }); - return true; - } catch (error) { - logger.warn({ - at: "OracleBot", - message: "Nonce backlog cancellation transaction failed", - attempt, - nonce: latestNonce, - error, - }); - } - } - - logger.error({ - at: "OracleBot", - message: "Failed to clear nonce backlog, exiting early", - nonce: latestNonce, - pendingNonce, - }); - - return true; -} async function main() { const params = await initMonitoringParams(process.env); @@ -146,20 +16,12 @@ async function main() { }); const gasEstimator = new GasEstimator(logger, undefined, params.chainId, params.provider); - const nonceBacklogConfig = getNonceBacklogConfig(process.env); const cmds = { settleRequestsEnabled: settleRequests, }; for (;;) { - const backlogDetected = await handleNonceBacklog(params, gasEstimator, nonceBacklogConfig); - if (backlogDetected) { - await delay(5); // Let any in-flight logs flush before exiting. - await waitForLogger(logger); - break; - } - await gasEstimator.update(); const runCmds = Object.entries(cmds) diff --git a/packages/monitor-v2/src/transaction-clearer/TransactionClearer.ts b/packages/monitor-v2/src/transaction-clearer/TransactionClearer.ts index e205162a15..74cdc887f0 100644 --- a/packages/monitor-v2/src/transaction-clearer/TransactionClearer.ts +++ b/packages/monitor-v2/src/transaction-clearer/TransactionClearer.ts @@ -11,24 +11,24 @@ function isLondonFeeData(feeData: FeeData): feeData is { maxFeePerGas: BigNumber } function bumpFeeData(baseFeeData: FeeData, attemptIndex: number, config: NonceBacklogConfig): FeeData { - // Calculate multiplier: (numerator/denominator)^(attemptIndex+1) - // For attempt 0: 1.2x, attempt 1: 1.44x, attempt 2: 1.73x (with default 12/10) - let numerator = BigNumber.from(config.feeBumpNumerator); - let denominator = BigNumber.from(config.feeBumpDenominator); - - for (let i = 0; i < attemptIndex; i++) { - numerator = numerator.mul(config.feeBumpNumerator); - denominator = denominator.mul(config.feeBumpDenominator); - } + // Calculate multiplier: ((100 + percent) / 100)^(attemptIndex+1) + // For attempt 0: 1.2x, attempt 1: 1.44x, attempt 2: 1.73x (with default 20%) + const bumpValue = (value: BigNumber): BigNumber => { + let bumped = value; + for (let i = 0; i <= attemptIndex; i++) { + bumped = bumped.mul(100 + config.feeBumpPercent).div(100); + } + return bumped; + }; if (isLondonFeeData(baseFeeData)) { return { - maxFeePerGas: baseFeeData.maxFeePerGas.mul(numerator).div(denominator), - maxPriorityFeePerGas: baseFeeData.maxPriorityFeePerGas.mul(numerator).div(denominator), + maxFeePerGas: bumpValue(baseFeeData.maxFeePerGas), + maxPriorityFeePerGas: bumpValue(baseFeeData.maxPriorityFeePerGas), }; } else { return { - gasPrice: baseFeeData.gasPrice.mul(numerator).div(denominator), + gasPrice: bumpValue(baseFeeData.gasPrice), }; } } diff --git a/packages/monitor-v2/src/transaction-clearer/common.ts b/packages/monitor-v2/src/transaction-clearer/common.ts index 3815de8841..c391a77f92 100644 --- a/packages/monitor-v2/src/transaction-clearer/common.ts +++ b/packages/monitor-v2/src/transaction-clearer/common.ts @@ -4,9 +4,8 @@ import { BaseMonitoringParams, initBaseMonitoringParams, startupLogLevel as base export interface NonceBacklogConfig { // Minimum nonce difference (pending - latest) to trigger clearing nonceBacklogThreshold: number; - // Fee bump multiplier: bumpedFee = fee * numerator / denominator - feeBumpNumerator: number; - feeBumpDenominator: number; + // Fee bump percentage per attempt (e.g., 20 means 20% increase) + feeBumpPercent: number; // Max attempts to replace a stuck transaction with increasing fees replacementAttempts: number; } @@ -15,14 +14,22 @@ export interface MonitoringParams extends BaseMonitoringParams { nonceBacklogConfig: NonceBacklogConfig; } +const parsePositiveInt = (value: string | undefined, defaultValue: number, name: string): number => { + if (value === undefined) return defaultValue; + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed <= 0 || !Number.isInteger(parsed)) { + throw new Error(`${name} must be a positive integer, got: ${value}`); + } + return parsed; +}; + export const initMonitoringParams = async (env: NodeJS.ProcessEnv): Promise => { const base = await initBaseMonitoringParams(env); const nonceBacklogConfig: NonceBacklogConfig = { - nonceBacklogThreshold: Number(env.NONCE_BACKLOG_THRESHOLD) || 1, - feeBumpNumerator: Number(env.FEE_BUMP_NUMERATOR) || 12, - feeBumpDenominator: Number(env.FEE_BUMP_DENOMINATOR) || 10, - replacementAttempts: Number(env.REPLACEMENT_ATTEMPTS) || 3, + nonceBacklogThreshold: parsePositiveInt(env.NONCE_BACKLOG_THRESHOLD, 1, "NONCE_BACKLOG_THRESHOLD"), + feeBumpPercent: parsePositiveInt(env.FEE_BUMP_PERCENT, 20, "FEE_BUMP_PERCENT"), + replacementAttempts: parsePositiveInt(env.REPLACEMENT_ATTEMPTS, 3, "REPLACEMENT_ATTEMPTS"), }; return { From dd0c8b790639da1264ba0c1e9ee4ff0d05ef5cb7 Mon Sep 17 00:00:00 2001 From: Pablo Maldonado Date: Mon, 19 Jan 2026 13:19:04 +0000 Subject: [PATCH 5/7] fix: use consistent NONCE_ prefix for env vars --- packages/monitor-v2/src/transaction-clearer/common.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/monitor-v2/src/transaction-clearer/common.ts b/packages/monitor-v2/src/transaction-clearer/common.ts index c391a77f92..3383cc20f3 100644 --- a/packages/monitor-v2/src/transaction-clearer/common.ts +++ b/packages/monitor-v2/src/transaction-clearer/common.ts @@ -28,8 +28,8 @@ export const initMonitoringParams = async (env: NodeJS.ProcessEnv): Promise Date: Mon, 19 Jan 2026 13:31:23 +0000 Subject: [PATCH 6/7] refactor: extract transaction clearing to shared bot-utils module Move clearStuckTransactions and related helpers to bot-utils/transactionClearing.ts so both transaction-clearer and bot-oo can reuse the same logic. Signed-off-by: Pablo Maldonado --- .../src/bot-utils/transactionClearing.ts | 206 ++++++++++++++++++ .../transaction-clearer/TransactionClearer.ts | 165 +------------- .../src/transaction-clearer/common.ts | 27 +-- 3 files changed, 211 insertions(+), 187 deletions(-) create mode 100644 packages/monitor-v2/src/bot-utils/transactionClearing.ts diff --git a/packages/monitor-v2/src/bot-utils/transactionClearing.ts b/packages/monitor-v2/src/bot-utils/transactionClearing.ts new file mode 100644 index 0000000000..3bd2d6fb4c --- /dev/null +++ b/packages/monitor-v2/src/bot-utils/transactionClearing.ts @@ -0,0 +1,206 @@ +import { BigNumber } from "ethers"; +import type { Logger as LoggerType } from "winston"; +import type { Provider } from "@ethersproject/abstract-provider"; +import type { Signer } from "ethers"; +import { GasEstimator } from "@uma/financial-templates-lib"; + +export interface NonceBacklogConfig { + // Minimum nonce difference (pending - latest) to trigger clearing + nonceBacklogThreshold: number; + // Fee bump percentage per attempt (e.g., 20 means 20% increase) + feeBumpPercent: number; + // Max attempts to replace a stuck transaction with increasing fees + replacementAttempts: number; +} + +export interface TransactionClearingParams { + provider: Provider; + signer: Signer; + nonceBacklogConfig: NonceBacklogConfig; +} + +type FeeData = { maxFeePerGas: BigNumber; maxPriorityFeePerGas: BigNumber } | { gasPrice: BigNumber }; + +function isLondonFeeData(feeData: FeeData): feeData is { maxFeePerGas: BigNumber; maxPriorityFeePerGas: BigNumber } { + return "maxFeePerGas" in feeData; +} + +export const parsePositiveInt = (value: string | undefined, defaultValue: number, name: string): number => { + if (value === undefined) return defaultValue; + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed <= 0 || !Number.isInteger(parsed)) { + throw new Error(`${name} must be a positive integer, got: ${value}`); + } + return parsed; +}; + +export const getNonceBacklogConfig = (env: NodeJS.ProcessEnv): NonceBacklogConfig => { + return { + nonceBacklogThreshold: parsePositiveInt(env.NONCE_BACKLOG_THRESHOLD, 1, "NONCE_BACKLOG_THRESHOLD"), + feeBumpPercent: parsePositiveInt(env.NONCE_REPLACEMENT_BUMP_PERCENT, 20, "NONCE_REPLACEMENT_BUMP_PERCENT"), + replacementAttempts: parsePositiveInt(env.NONCE_REPLACEMENT_ATTEMPTS, 3, "NONCE_REPLACEMENT_ATTEMPTS"), + }; +}; + +function bumpFeeData(baseFeeData: FeeData, attemptIndex: number, config: NonceBacklogConfig): FeeData { + // Calculate multiplier: ((100 + percent) / 100)^(attemptIndex+1) + // For attempt 0: 1.2x, attempt 1: 1.44x, attempt 2: 1.73x (with default 20%) + const bumpValue = (value: BigNumber): BigNumber => { + let bumped = value; + for (let i = 0; i <= attemptIndex; i++) { + bumped = bumped.mul(100 + config.feeBumpPercent).div(100); + } + return bumped; + }; + + if (isLondonFeeData(baseFeeData)) { + return { + maxFeePerGas: bumpValue(baseFeeData.maxFeePerGas), + maxPriorityFeePerGas: bumpValue(baseFeeData.maxPriorityFeePerGas), + }; + } else { + return { + gasPrice: bumpValue(baseFeeData.gasPrice), + }; + } +} + +async function getNonces(provider: Provider, address: string): Promise<{ latestNonce: number; pendingNonce: number }> { + const [latestNonce, pendingNonce] = await Promise.all([ + provider.getTransactionCount(address, "latest"), + provider.getTransactionCount(address, "pending"), + ]); + return { latestNonce, pendingNonce }; +} + +/** + * Clears stuck transactions by sending self-transactions with higher gas fees. + * @returns true if a nonce backlog was detected and clearing was attempted + */ +export async function clearStuckTransactions( + logger: LoggerType, + params: TransactionClearingParams, + gasEstimator: GasEstimator +): Promise { + const { provider, signer, nonceBacklogConfig } = params; + const botAddress = await signer.getAddress(); + + const { latestNonce, pendingNonce } = await getNonces(provider, botAddress); + const backlog = pendingNonce - latestNonce; + + if (backlog < nonceBacklogConfig.nonceBacklogThreshold) { + logger.debug({ + at: "TransactionClearer", + message: "No nonce backlog detected", + botAddress, + latestNonce, + pendingNonce, + backlog, + threshold: nonceBacklogConfig.nonceBacklogThreshold, + }); + return false; + } + + logger.warn({ + at: "TransactionClearer", + message: "Nonce backlog detected, attempting to clear stuck transactions", + botAddress, + latestNonce, + pendingNonce, + backlog, + threshold: nonceBacklogConfig.nonceBacklogThreshold, + }); + + // Get base fee data from gas estimator + const baseFeeData = gasEstimator.getCurrentFastPriceEthers(); + + // Clear all stuck nonces from latestNonce to pendingNonce - 1 + for (let nonce = latestNonce; nonce < pendingNonce; nonce++) { + let cleared = false; + + for (let attempt = 0; attempt < nonceBacklogConfig.replacementAttempts; attempt++) { + const feeData = bumpFeeData(baseFeeData, attempt, nonceBacklogConfig); + + try { + logger.info({ + at: "TransactionClearer", + message: `Attempting to clear stuck transaction (nonce ${nonce}, attempt ${attempt + 1})`, + botAddress, + nonce, + attempt: attempt + 1, + feeData: isLondonFeeData(feeData) + ? { + maxFeePerGas: feeData.maxFeePerGas.toString(), + maxPriorityFeePerGas: feeData.maxPriorityFeePerGas.toString(), + } + : { gasPrice: feeData.gasPrice.toString() }, + }); + + const tx = await signer.sendTransaction({ + to: botAddress, // Self-transaction + value: 0, + nonce, + gasLimit: 21_000, + ...feeData, + }); + + const receipt = await tx.wait(1); + + logger.info({ + at: "TransactionClearer", + message: `Successfully cleared stuck transaction (nonce ${nonce})`, + botAddress, + nonce, + transactionHash: receipt.transactionHash, + gasUsed: receipt.gasUsed.toString(), + }); + + cleared = true; + break; + } catch (error) { + logger.warn({ + at: "TransactionClearer", + message: `Failed to clear stuck transaction (nonce ${nonce}, attempt ${attempt + 1})`, + botAddress, + nonce, + attempt: attempt + 1, + error: error instanceof Error ? error.message : String(error), + }); + } + } + + if (!cleared) { + logger.error({ + at: "TransactionClearer", + message: `Failed to clear stuck transaction after all attempts (nonce ${nonce})`, + botAddress, + nonce, + maxAttempts: nonceBacklogConfig.replacementAttempts, + }); + } + } + + // Verify final state + const { latestNonce: finalLatestNonce, pendingNonce: finalPendingNonce } = await getNonces(provider, botAddress); + const finalBacklog = finalPendingNonce - finalLatestNonce; + + if (finalBacklog < nonceBacklogConfig.nonceBacklogThreshold) { + logger.info({ + at: "TransactionClearer", + message: "Successfully cleared nonce backlog", + botAddress, + previousBacklog: backlog, + finalBacklog, + }); + } else { + logger.warn({ + at: "TransactionClearer", + message: "Nonce backlog still present after clearing attempt", + botAddress, + previousBacklog: backlog, + finalBacklog, + }); + } + + return true; +} diff --git a/packages/monitor-v2/src/transaction-clearer/TransactionClearer.ts b/packages/monitor-v2/src/transaction-clearer/TransactionClearer.ts index 74cdc887f0..64c2703759 100644 --- a/packages/monitor-v2/src/transaction-clearer/TransactionClearer.ts +++ b/packages/monitor-v2/src/transaction-clearer/TransactionClearer.ts @@ -1,171 +1,12 @@ -import { BigNumber } from "ethers"; import type { Logger as LoggerType } from "winston"; -import type { Provider } from "@ethersproject/abstract-provider"; import { GasEstimator } from "@uma/financial-templates-lib"; -import { MonitoringParams, NonceBacklogConfig } from "./common"; - -type FeeData = { maxFeePerGas: BigNumber; maxPriorityFeePerGas: BigNumber } | { gasPrice: BigNumber }; - -function isLondonFeeData(feeData: FeeData): feeData is { maxFeePerGas: BigNumber; maxPriorityFeePerGas: BigNumber } { - return "maxFeePerGas" in feeData; -} - -function bumpFeeData(baseFeeData: FeeData, attemptIndex: number, config: NonceBacklogConfig): FeeData { - // Calculate multiplier: ((100 + percent) / 100)^(attemptIndex+1) - // For attempt 0: 1.2x, attempt 1: 1.44x, attempt 2: 1.73x (with default 20%) - const bumpValue = (value: BigNumber): BigNumber => { - let bumped = value; - for (let i = 0; i <= attemptIndex; i++) { - bumped = bumped.mul(100 + config.feeBumpPercent).div(100); - } - return bumped; - }; - - if (isLondonFeeData(baseFeeData)) { - return { - maxFeePerGas: bumpValue(baseFeeData.maxFeePerGas), - maxPriorityFeePerGas: bumpValue(baseFeeData.maxPriorityFeePerGas), - }; - } else { - return { - gasPrice: bumpValue(baseFeeData.gasPrice), - }; - } -} - -async function getNonces( - provider: Provider, - address: string -): Promise<{ latestNonce: number; pendingNonce: number }> { - const [latestNonce, pendingNonce] = await Promise.all([ - provider.getTransactionCount(address, "latest"), - provider.getTransactionCount(address, "pending"), - ]); - return { latestNonce, pendingNonce }; -} +import { MonitoringParams } from "./common"; +import { clearStuckTransactions as clearStuckTransactionsImpl } from "../bot-utils/transactionClearing"; export async function clearStuckTransactions( logger: LoggerType, params: MonitoringParams, gasEstimator: GasEstimator ): Promise { - const { provider, signer, nonceBacklogConfig } = params; - const botAddress = await signer.getAddress(); - - const { latestNonce, pendingNonce } = await getNonces(provider, botAddress); - const backlog = pendingNonce - latestNonce; - - if (backlog < nonceBacklogConfig.nonceBacklogThreshold) { - logger.debug({ - at: "TransactionClearer", - message: "No nonce backlog detected", - botAddress, - latestNonce, - pendingNonce, - backlog, - threshold: nonceBacklogConfig.nonceBacklogThreshold, - }); - return; - } - - logger.warn({ - at: "TransactionClearer", - message: "Nonce backlog detected, attempting to clear stuck transactions", - botAddress, - latestNonce, - pendingNonce, - backlog, - threshold: nonceBacklogConfig.nonceBacklogThreshold, - }); - - // Get base fee data from gas estimator - const baseFeeData = gasEstimator.getCurrentFastPriceEthers(); - - // Clear all stuck nonces from latestNonce to pendingNonce - 1 - for (let nonce = latestNonce; nonce < pendingNonce; nonce++) { - let cleared = false; - - for (let attempt = 0; attempt < nonceBacklogConfig.replacementAttempts; attempt++) { - const feeData = bumpFeeData(baseFeeData, attempt, nonceBacklogConfig); - - try { - logger.info({ - at: "TransactionClearer", - message: `Attempting to clear stuck transaction (nonce ${nonce}, attempt ${attempt + 1})`, - botAddress, - nonce, - attempt: attempt + 1, - feeData: isLondonFeeData(feeData) - ? { - maxFeePerGas: feeData.maxFeePerGas.toString(), - maxPriorityFeePerGas: feeData.maxPriorityFeePerGas.toString(), - } - : { gasPrice: feeData.gasPrice.toString() }, - }); - - const tx = await signer.sendTransaction({ - to: botAddress, // Self-transaction - value: 0, - nonce, - gasLimit: 21_000, - ...feeData, - }); - - const receipt = await tx.wait(1); - - logger.info({ - at: "TransactionClearer", - message: `Successfully cleared stuck transaction (nonce ${nonce})`, - botAddress, - nonce, - transactionHash: receipt.transactionHash, - gasUsed: receipt.gasUsed.toString(), - }); - - cleared = true; - break; - } catch (error) { - logger.warn({ - at: "TransactionClearer", - message: `Failed to clear stuck transaction (nonce ${nonce}, attempt ${attempt + 1})`, - botAddress, - nonce, - attempt: attempt + 1, - error: error instanceof Error ? error.message : String(error), - }); - } - } - - if (!cleared) { - logger.error({ - at: "TransactionClearer", - message: `Failed to clear stuck transaction after all attempts (nonce ${nonce})`, - botAddress, - nonce, - maxAttempts: nonceBacklogConfig.replacementAttempts, - }); - } - } - - // Verify final state - const { latestNonce: finalLatestNonce, pendingNonce: finalPendingNonce } = await getNonces(provider, botAddress); - const finalBacklog = finalPendingNonce - finalLatestNonce; - - if (finalBacklog < nonceBacklogConfig.nonceBacklogThreshold) { - logger.info({ - at: "TransactionClearer", - message: "Successfully cleared nonce backlog", - botAddress, - previousBacklog: backlog, - finalBacklog, - }); - } else { - logger.warn({ - at: "TransactionClearer", - message: "Nonce backlog still present after clearing attempt", - botAddress, - previousBacklog: backlog, - finalBacklog, - }); - } + await clearStuckTransactionsImpl(logger, params, gasEstimator); } diff --git a/packages/monitor-v2/src/transaction-clearer/common.ts b/packages/monitor-v2/src/transaction-clearer/common.ts index 3383cc20f3..cc66a00168 100644 --- a/packages/monitor-v2/src/transaction-clearer/common.ts +++ b/packages/monitor-v2/src/transaction-clearer/common.ts @@ -1,40 +1,17 @@ export { Logger } from "@uma/financial-templates-lib"; import { BaseMonitoringParams, initBaseMonitoringParams, startupLogLevel as baseStartup } from "../bot-utils/base"; - -export interface NonceBacklogConfig { - // Minimum nonce difference (pending - latest) to trigger clearing - nonceBacklogThreshold: number; - // Fee bump percentage per attempt (e.g., 20 means 20% increase) - feeBumpPercent: number; - // Max attempts to replace a stuck transaction with increasing fees - replacementAttempts: number; -} +import { getNonceBacklogConfig, NonceBacklogConfig } from "../bot-utils/transactionClearing"; export interface MonitoringParams extends BaseMonitoringParams { nonceBacklogConfig: NonceBacklogConfig; } -const parsePositiveInt = (value: string | undefined, defaultValue: number, name: string): number => { - if (value === undefined) return defaultValue; - const parsed = Number(value); - if (!Number.isFinite(parsed) || parsed <= 0 || !Number.isInteger(parsed)) { - throw new Error(`${name} must be a positive integer, got: ${value}`); - } - return parsed; -}; - export const initMonitoringParams = async (env: NodeJS.ProcessEnv): Promise => { const base = await initBaseMonitoringParams(env); - const nonceBacklogConfig: NonceBacklogConfig = { - nonceBacklogThreshold: parsePositiveInt(env.NONCE_BACKLOG_THRESHOLD, 1, "NONCE_BACKLOG_THRESHOLD"), - feeBumpPercent: parsePositiveInt(env.NONCE_REPLACEMENT_BUMP_PERCENT, 20, "NONCE_REPLACEMENT_BUMP_PERCENT"), - replacementAttempts: parsePositiveInt(env.NONCE_REPLACEMENT_ATTEMPTS, 3, "NONCE_REPLACEMENT_ATTEMPTS"), - }; - return { ...base, - nonceBacklogConfig, + nonceBacklogConfig: getNonceBacklogConfig(env), }; }; From e706178ee60389952ec2945e1f17ec18fefbef50 Mon Sep 17 00:00:00 2001 From: Pablo Maldonado Date: Mon, 19 Jan 2026 13:55:08 +0000 Subject: [PATCH 7/7] chore: add comment to transaction-clearer Signed-off-by: Pablo Maldonado --- packages/monitor-v2/src/transaction-clearer/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/monitor-v2/src/transaction-clearer/index.ts b/packages/monitor-v2/src/transaction-clearer/index.ts index 7214eb3e2c..af20dda78c 100644 --- a/packages/monitor-v2/src/transaction-clearer/index.ts +++ b/packages/monitor-v2/src/transaction-clearer/index.ts @@ -1,3 +1,4 @@ +// Standalone bot for clearing stuck transactions via self-tx replacement. import { delay, waitForLogger, GasEstimator } from "@uma/financial-templates-lib"; import { initMonitoringParams, Logger, startupLogLevel } from "./common"; import { clearStuckTransactions } from "./TransactionClearer";