From 28e98470cd96210d85d2fb7ab024c9bff5abdf61 Mon Sep 17 00:00:00 2001 From: Daniel Padrino Date: Wed, 29 Jul 2026 12:33:33 -0300 Subject: [PATCH 1/3] Collapse Telegram polling-error noise into onset and recovery lines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without a 'polling_error' listener node-telegram-bot-api writes its own unformatted console error for every failed poll, at error level and outside the application logger. The polling loop also retries on a fixed 300ms interval with no backoff, so a brief Telegram gateway outage produces several lines per second — a recent nine-second 502 window logged 27 lines — and sustained retries earn a 429 on top of the original 502 because the loop ignores Telegram's retry-after. Attach a listener that reports one line per distinct failure plus one on recovery with duration and attempt count. The same outage becomes two lines, an escalation from 502 to 429 is still surfaced, and an outage never appears to stay open. Both lines are logged at warn so they remain visible together under a warn-level log configuration. No unit tests added: the repo's jest config has rootDir="src" but the sources live at the repo root, so `yarn test` finds zero tests today. --- socialmedia/telegram/telegram.service.ts | 43 ++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/socialmedia/telegram/telegram.service.ts b/socialmedia/telegram/telegram.service.ts index ca6cc86..d61e9f3 100644 --- a/socialmedia/telegram/telegram.service.ts +++ b/socialmedia/telegram/telegram.service.ts @@ -35,6 +35,9 @@ import { TelegramGroupState, TelegramState } from './telegram.types'; // Stay under telegram per-chat rate limit (~30 msg/s) when bursting position-lifecycle alerts. const TELEGRAM_THROTTLE_MS = 100; +// Consider polling healthy again once no further error arrived for this long. +const POLLING_RECOVERY_GRACE_MS = 30_000; + @Injectable() export class TelegramService implements OnModuleInit, SocialMediaFct { private readonly logger = new Logger(this.constructor.name); @@ -42,6 +45,8 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { private readonly telegramHandles: string[] = ['/start', '/subscribe', '/unsubscribe', '/help']; private readonly telegramState: TelegramState; private telegramGroupState: TelegramGroupState; + private pollingOutage: { since: number; lastAt: number; attempts: number; signature: string } | null = null; + private pollingRecoveryTimer: NodeJS.Timeout | null = null; constructor( private readonly socialMediaService: SocialMediaService, @@ -51,6 +56,10 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { private readonly position: PositionsService, private readonly challenge: ChallengesService ) { + // Without a 'polling_error' listener node-telegram-bot-api writes its own unformatted + // console error for every failed poll, and it retries on a fixed interval with no backoff. + this.bot.on('polling_error', (error) => this.onPollingError(error)); + const time: number = Date.now() + 365 * 24 * 60 * 60 * 1000; this.telegramState = { @@ -363,6 +372,40 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { } } + // Telegram's gateway answers 502/504 during its own restarts, so a short outage would + // otherwise produce several identical lines per second. Report one line per distinct + // failure and one on recovery, both at warn so an outage never appears to stay open. + private onPollingError(error: Error & { code?: string }): void { + const signature = error?.message ?? String(error); + const now = Date.now(); + + if (!this.pollingOutage) { + this.pollingOutage = { since: now, lastAt: now, attempts: 0, signature: '' }; + } + + this.pollingOutage.attempts++; + this.pollingOutage.lastAt = now; + + if (this.pollingOutage.signature !== signature) { + this.pollingOutage.signature = signature; + this.logger.warn(`Telegram polling failing: ${signature}`); + } + + if (this.pollingRecoveryTimer) clearTimeout(this.pollingRecoveryTimer); + this.pollingRecoveryTimer = setTimeout(() => this.onPollingRecovered(), POLLING_RECOVERY_GRACE_MS); + this.pollingRecoveryTimer.unref(); + } + + private onPollingRecovered(): void { + if (!this.pollingOutage) return; + + const { since, lastAt, attempts } = this.pollingOutage; + this.pollingOutage = null; + this.pollingRecoveryTimer = null; + + this.logger.warn(`Telegram polling recovered after ${Math.round((lastAt - since) / 1000)}s and ${attempts} failed attempts`); + } + private async doSendMessage(group: string | number, message: string): Promise { await this.bot.sendMessage(group.toString(), message, { parse_mode: 'Markdown', disable_web_page_preview: true }); } From 84b1c73d23cf82f19d74828e62258a8b62c9b647 Mon Sep 17 00:00:00 2001 From: Daniel Padrino Date: Wed, 29 Jul 2026 12:46:02 -0300 Subject: [PATCH 2/3] Keep unrecovered polling outages visible in the logs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up on the first commit. The signature used the raw error message compared against the previous one only, which fails in the two cases that actually occur: "Too Many Requests: retry after N" counts down and a connect failure carries a rotating gateway address, so each retry looked like a new failure, and alternating errors re-reported on every poll. Mask digits out of the signature and track the signatures already reported within the current outage. A permanently failing poll — a revoked token answers 401 forever — reported once and then stayed silent, because the recovery timer only fires once errors stop. Repeat the report every five minutes while the outage is open. Also move the outage shape into telegram.types.ts next to the other state types, and truncate the error text, which wraps the upstream response body. --- socialmedia/telegram/telegram.service.ts | 25 +++++++++++++++++------- socialmedia/telegram/telegram.types.ts | 10 ++++++++++ 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/socialmedia/telegram/telegram.service.ts b/socialmedia/telegram/telegram.service.ts index d61e9f3..654367b 100644 --- a/socialmedia/telegram/telegram.service.ts +++ b/socialmedia/telegram/telegram.service.ts @@ -30,13 +30,17 @@ import { PositionProposalMessage } from './messages/PositionProposal.message'; import { SavingUpdateMessage } from './messages/SavingUpdate.message'; import { StablecoinBridgeMessage } from './messages/StablecoinBridgeUpdate.message'; import { TradeMessage } from './messages/Trade.message'; -import { TelegramGroupState, TelegramState } from './telegram.types'; +import { TelegramGroupState, TelegramPollingOutage, TelegramState } from './telegram.types'; // Stay under telegram per-chat rate limit (~30 msg/s) when bursting position-lifecycle alerts. const TELEGRAM_THROTTLE_MS = 100; // Consider polling healthy again once no further error arrived for this long. const POLLING_RECOVERY_GRACE_MS = 30_000; +// Repeat an ongoing outage at this interval so a permanent failure never goes silent. +const POLLING_REPORT_INTERVAL_MS = 300_000; +// Telegram errors wrap the upstream response body, which is not always short. +const MAX_POLLING_ERROR_LENGTH = 200; @Injectable() export class TelegramService implements OnModuleInit, SocialMediaFct { @@ -45,7 +49,7 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { private readonly telegramHandles: string[] = ['/start', '/subscribe', '/unsubscribe', '/help']; private readonly telegramState: TelegramState; private telegramGroupState: TelegramGroupState; - private pollingOutage: { since: number; lastAt: number; attempts: number; signature: string } | null = null; + private pollingOutage: TelegramPollingOutage | null = null; private pollingRecoveryTimer: NodeJS.Timeout | null = null; constructor( @@ -376,19 +380,26 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { // otherwise produce several identical lines per second. Report one line per distinct // failure and one on recovery, both at warn so an outage never appears to stay open. private onPollingError(error: Error & { code?: string }): void { - const signature = error?.message ?? String(error); + const message = (error?.message ?? String(error)).slice(0, MAX_POLLING_ERROR_LENGTH); + // Digits carry the volatile parts of a telegram error — the countdown in + // "retry after 5" and the rotating gateway address in a connect failure — so they are + // masked out of the signature to keep one repeating failure on one line. + const signature = `${error?.code ?? 'UNKNOWN'}: ${message.replace(/\d+/g, '#')}`; const now = Date.now(); if (!this.pollingOutage) { - this.pollingOutage = { since: now, lastAt: now, attempts: 0, signature: '' }; + this.pollingOutage = { since: now, lastAt: now, lastReportAt: 0, attempts: 0, signatures: new Set() }; } this.pollingOutage.attempts++; this.pollingOutage.lastAt = now; - if (this.pollingOutage.signature !== signature) { - this.pollingOutage.signature = signature; - this.logger.warn(`Telegram polling failing: ${signature}`); + const isNewFailure = !this.pollingOutage.signatures.has(signature); + if (isNewFailure) this.pollingOutage.signatures.add(signature); + + if (isNewFailure || now - this.pollingOutage.lastReportAt >= POLLING_REPORT_INTERVAL_MS) { + this.pollingOutage.lastReportAt = now; + this.logger.warn(`Telegram polling failing (attempt ${this.pollingOutage.attempts}): ${message}`); } if (this.pollingRecoveryTimer) clearTimeout(this.pollingRecoveryTimer); diff --git a/socialmedia/telegram/telegram.types.ts b/socialmedia/telegram/telegram.types.ts index 4158b4c..6cb09bf 100644 --- a/socialmedia/telegram/telegram.types.ts +++ b/socialmedia/telegram/telegram.types.ts @@ -19,6 +19,16 @@ export type TelegramSubscriptionState = { tradeUpdates: number; }; +// @dev: in-memory state of an ongoing telegram polling outage, used to report it once +// per distinct failure instead of once per retry +export type TelegramPollingOutage = { + since: number; + lastAt: number; + lastReportAt: number; + attempts: number; + signatures: Set; +}; + export type TelegramGroupState = { apiVersion: string; createdAt: number; From d08977b59d6434608065c543283ce42e98784397 Mon Sep 17 00:00:00 2001 From: Daniel Padrino Date: Wed, 29 Jul 2026 12:57:27 -0300 Subject: [PATCH 3/3] Bound the outage signature set and stop over-claiming recovery Second review follow-up. The signature prefixed the error code onto a message that the library already prefixes with that same code, so the prefix discriminated nothing. Use the message alone. The signature set had no upper bound: a message carrying a per-attempt token that survives digit masking - a request id in an upstream error body reaches the parse-error branch verbatim - produced a fresh signature per poll, which defeated the collapsing and grew the set without limit. Cap it, past which only the periodic report remains. A replay of 2000 such polls now yields 22 lines and 20 retained signatures instead of 2000 of each. An error arriving more than the grace period after the previous one opened and closed its own outage, so an isolated blip cost two lines where it used to cost one. Skip the closing line for a single attempt. The closing line said "recovered", but nothing probes the poll - the grace period only establishes that no further error arrived, and a request that never settles would look the same. Say what is actually known instead. --- socialmedia/telegram/telegram.service.ts | 26 ++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/socialmedia/telegram/telegram.service.ts b/socialmedia/telegram/telegram.service.ts index 654367b..e7b8d77 100644 --- a/socialmedia/telegram/telegram.service.ts +++ b/socialmedia/telegram/telegram.service.ts @@ -41,6 +41,8 @@ const POLLING_RECOVERY_GRACE_MS = 30_000; const POLLING_REPORT_INTERVAL_MS = 300_000; // Telegram errors wrap the upstream response body, which is not always short. const MAX_POLLING_ERROR_LENGTH = 200; +// Beyond this many distinct failures one outage reports on its interval only. +const MAX_POLLING_SIGNATURES = 20; @Injectable() export class TelegramService implements OnModuleInit, SocialMediaFct { @@ -378,13 +380,15 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { // Telegram's gateway answers 502/504 during its own restarts, so a short outage would // otherwise produce several identical lines per second. Report one line per distinct - // failure and one on recovery, both at warn so an outage never appears to stay open. - private onPollingError(error: Error & { code?: string }): void { + // failure, repeat an outage that never clears every POLLING_REPORT_INTERVAL_MS, and close + // it with one line — all at warn, so an outage never appears to stay open. + private onPollingError(error: Error): void { const message = (error?.message ?? String(error)).slice(0, MAX_POLLING_ERROR_LENGTH); - // Digits carry the volatile parts of a telegram error — the countdown in - // "retry after 5" and the rotating gateway address in a connect failure — so they are - // masked out of the signature to keep one repeating failure on one line. - const signature = `${error?.code ?? 'UNKNOWN'}: ${message.replace(/\d+/g, '#')}`; + // The library prefixes its own error code onto the message, so the message alone + // identifies the failure. Digits carry the volatile parts — the countdown in + // "retry after 5", the rotating gateway address in a connect failure — and are masked + // out to keep a repeating failure on one line. + const signature = message.replace(/\d+/g, '#'); const now = Date.now(); if (!this.pollingOutage) { @@ -394,7 +398,9 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { this.pollingOutage.attempts++; this.pollingOutage.lastAt = now; - const isNewFailure = !this.pollingOutage.signatures.has(signature); + // Past the cap only the periodic report remains, which bounds the log volume and the + // set itself if a message carries a token that survives digit masking. + const isNewFailure = !this.pollingOutage.signatures.has(signature) && this.pollingOutage.signatures.size < MAX_POLLING_SIGNATURES; if (isNewFailure) this.pollingOutage.signatures.add(signature); if (isNewFailure || now - this.pollingOutage.lastReportAt >= POLLING_REPORT_INTERVAL_MS) { @@ -414,7 +420,11 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { this.pollingOutage = null; this.pollingRecoveryTimer = null; - this.logger.warn(`Telegram polling recovered after ${Math.round((lastAt - since) / 1000)}s and ${attempts} failed attempts`); + // A single failure needs no closing line. Silence only proves that no further error + // arrived — the poll itself is not probed here, so the line does not claim more. + if (attempts <= 1) return; + + this.logger.warn(`Telegram polling errors stopped after ${Math.round((lastAt - since) / 1000)}s and ${attempts} attempts`); } private async doSendMessage(group: string | number, message: string): Promise {