From 07ff70e72570ccdcdb0010b689c1ffcc76b7f5ce Mon Sep 17 00:00:00 2001 From: Danswar <48102227+Danswar@users.noreply.github.com> Date: Wed, 29 Jul 2026 15:08:12 -0300 Subject: [PATCH] Fall back to text-only posts and collapse Telegram polling noise (#66) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fall back to text-only posts and collapse Telegram polling noise TELEGRAM_IMAGES_DIR is unset in the deployed environment, so every video-bearing notification builds the literal path "undefined/.mp4". node-telegram-bot-api reinterprets a path that is not a readable file as a URL, Telegram answers "400 Bad Request: wrong HTTP URL specified", and sendMessage's catch only logs a warning — the notification is dropped. Trade, MintingUpdate, SavingUpdate, StablecoinBridgeUpdate and FrontendCodeRegistered have therefore never reached subscribers, while text-only alerts kept working and made the bot look healthy. Setting the env var alone would not help: the media files are not in the image. Port the guard d-EURO/api#115 already uses: resolveMediaPath checks the file with existsSync and returns undefined, so sendMessage degrades to sendMessage instead of sendVideo. Separately, attach a 'polling_error' listener. Without one the library writes its own unformatted console error for every failed poll, and it retries on a fixed interval with no backoff — a nine-second Telegram gateway outage produced 27 error lines, and sustained retries earn a 429 on top of the original 502. The listener reports one line per distinct failure plus one on recovery with duration and attempt count, which turns the same outage into two lines while still surfacing an escalation from 502 to 429. Both lines are logged at warn so an outage never appears to stay open under a warn-level log configuration. No unit tests added: the repo's jest config has rootDir="src" but the sources live at the repo root, so `yarn test` finds zero tests today. * Keep missing assets and unrecovered outages visible in the logs Review follow-up on the first two commits. The missing-asset notice was logged at debug, which the deployed logger never emits. Since the assets are absent from the image the fallback is the permanent state, so the condition would have become completely silent — it was visible as a warning before this branch. Report it at warn, once per distinct asset per boot. The polling signature used the raw error message compared against the previous one only, which fails in the two cases that actually occur: "Too Many Requests: retry after N" counts down and a connect failure carries a rotating gateway address, so each retry looked like a new failure, and alternating errors re-reported on every poll. Mask digits out of the signature and track the signatures already reported within the current outage. A permanently failing poll — a revoked token answers 401 forever — reported once and then stayed silent, because the recovery timer only fires once errors stop. Repeat the report every five minutes while the outage is open. Also apply the same media guard to TwitterService, which is the other caller of the shared helper and logged an upload error per post, move the outage shape into telegram.types.ts next to the other state types, and truncate the error text, which wraps the upstream response body. * Bound the outage signature set and stop over-claiming recovery Second review follow-up. The signature prefixed the error code onto a message that the library already prefixes with that same code, so the prefix discriminated nothing. Use the message alone. The signature set had no upper bound: a message carrying a per-attempt token that survives digit masking - a request id in an upstream error body reaches the parse-error branch verbatim - produced a fresh signature per poll, which defeated the collapsing and grew the set without limit. Cap it, past which only the periodic report remains. A replay of 2000 such polls now yields 22 lines and 20 retained signatures instead of 2000 of each. An error arriving more than the grace period after the previous one opened and closed its own outage, so an isolated blip cost two lines where it used to cost one. Skip the closing line for a single attempt. The closing line said "recovered", but nothing probes the poll - the grace period only establishes that no further error arrived, and a request that never settles would look the same. Say what is actually known instead. --- socialmedia/socialmedia.helper.ts | 8 +++ socialmedia/telegram/telegram.service.ts | 79 +++++++++++++++++++++++- socialmedia/telegram/telegram.types.ts | 10 +++ socialmedia/twitter/twitter.service.ts | 14 ++++- 4 files changed, 107 insertions(+), 4 deletions(-) diff --git a/socialmedia/socialmedia.helper.ts b/socialmedia/socialmedia.helper.ts index 9cfc532..cbfe6a3 100644 --- a/socialmedia/socialmedia.helper.ts +++ b/socialmedia/socialmedia.helper.ts @@ -1,7 +1,15 @@ +import { existsSync } from 'fs'; import { Hex, hexToString } from 'viem'; const JUICE_WALLET_FRONTEND_CODE = '0xe8d44050873dba865aa7c170ab4cce64d90839a34dcfd6cf71d14e0205443b1b'; +// Returns the input path when the asset is readable on disk, otherwise undefined. +// Missing notification assets must not block posting — callers fall back to text-only. +export function resolveMediaPath(media: string | undefined): string | undefined { + if (!media) return undefined; + return existsSync(media) ? media : undefined; +} + export function createRefCode(frontendCode: string): string | undefined { if (frontendCode?.startsWith('0x00')) { return hexToString(frontendCode as Hex).replace(/[\x00-\x1f,\x7f]/g, ''); diff --git a/socialmedia/telegram/telegram.service.ts b/socialmedia/telegram/telegram.service.ts index 915625e..5e22979 100644 --- a/socialmedia/telegram/telegram.service.ts +++ b/socialmedia/telegram/telegram.service.ts @@ -8,6 +8,7 @@ import { FrontendCodeRegisteredQuery, FrontendCodeSavingsQuery } from 'frontendc import TelegramBot from 'node-telegram-bot-api'; import { PositionsService } from 'positions/positions.service'; import { SavingsLeadrateService } from 'savings/savings.leadrate.service'; +import { resolveMediaPath } from 'socialmedia/socialmedia.helper'; import { SocialMediaFct, SocialMediaService } from 'socialmedia/socialmedia.service'; import { StorageService } from 'storage/storage.service'; import { TradeQuery } from 'trades/trade.types'; @@ -25,7 +26,16 @@ import { PositionProposalMessage } from './messages/PositionProposal.message'; import { SavingUpdateMessage } from './messages/SavingUpdate.message'; import { StablecoinBridgeMessage } from './messages/StablecoinBridgeUpdate.message'; import { TradeMessage } from './messages/Trade.message'; -import { TelegramGroupState, TelegramState } from './telegram.types'; +import { TelegramGroupState, TelegramPollingOutage, TelegramState } from './telegram.types'; + +// Consider polling healthy again once no further error arrived for this long. +const POLLING_RECOVERY_GRACE_MS = 30_000; +// Repeat an ongoing outage at this interval so a permanent failure never goes silent. +const POLLING_REPORT_INTERVAL_MS = 300_000; +// Telegram errors wrap the upstream response body, which is not always short. +const MAX_POLLING_ERROR_LENGTH = 200; +// Beyond this many distinct failures one outage reports on its interval only. +const MAX_POLLING_SIGNATURES = 20; @Injectable() export class TelegramService implements OnModuleInit, SocialMediaFct { @@ -34,6 +44,9 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { private readonly telegramHandles: string[] = ['/start', '/subscribe', '/unsubscribe', '/help']; private readonly telegramState: TelegramState; private telegramGroupState: TelegramGroupState; + private pollingOutage: TelegramPollingOutage | null = null; + private pollingRecoveryTimer: NodeJS.Timeout | null = null; + private readonly reportedMissingMedia = new Set(); constructor( private readonly socialMediaService: SocialMediaService, @@ -44,6 +57,9 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { private readonly challenge: ChallengesService ) { this.bot = CONFIG.telegram ? new TelegramBot(CONFIG.telegram.botToken, { polling: true }) : null; + // Without a 'polling_error' listener node-telegram-bot-api writes its own unformatted + // console error for every failed poll, and it retries on a fixed interval with no backoff. + this.bot?.on('polling_error', (error) => this.onPollingError(error)); const time: number = Date.now() + 365 * 24 * 60 * 60 * 1000; @@ -241,7 +257,17 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { private async sendMessage(group: string | number, message: string, video?: string) { try { this.logger.log(`Sending message to group id: ${group}`); - video ? await this.doSendVideo(group, message, video) : await this.doSendMessage(group, message); + // Notification assets are best-effort — when the file is missing fall back to a + // text-only message rather than letting node-telegram-bot-api reinterpret a missing + // local path as a URL (which produces "wrong HTTP URL specified" 400s on every send). + const videoPath = resolveMediaPath(video); + if (video && !videoPath && !this.reportedMissingMedia.has(video)) { + // Once per asset per boot: a missing asset is a deployment defect that never + // heals on its own, so it must stay visible without repeating on every send. + this.reportedMissingMedia.add(video); + this.logger.warn(`Telegram video asset missing: ${video} — sending text-only`); + } + videoPath ? await this.doSendVideo(group, message, videoPath) : await this.doSendMessage(group, message); } catch (error) { const msg = { notFound: 'chat not found', @@ -268,6 +294,55 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { } } + // Telegram's gateway answers 502/504 during its own restarts, so a short outage would + // otherwise produce several identical lines per second. Report one line per distinct + // failure, repeat an outage that never clears every POLLING_REPORT_INTERVAL_MS, and close + // it with one line — all at warn, so an outage never appears to stay open. + private onPollingError(error: Error): void { + const message = (error?.message ?? String(error)).slice(0, MAX_POLLING_ERROR_LENGTH); + // The library prefixes its own error code onto the message, so the message alone + // identifies the failure. Digits carry the volatile parts — the countdown in + // "retry after 5", the rotating gateway address in a connect failure — and are masked + // out to keep a repeating failure on one line. + const signature = message.replace(/\d+/g, '#'); + const now = Date.now(); + + if (!this.pollingOutage) { + this.pollingOutage = { since: now, lastAt: now, lastReportAt: 0, attempts: 0, signatures: new Set() }; + } + + this.pollingOutage.attempts++; + this.pollingOutage.lastAt = now; + + // Past the cap only the periodic report remains, which bounds the log volume and the + // set itself if a message carries a token that survives digit masking. + const isNewFailure = !this.pollingOutage.signatures.has(signature) && this.pollingOutage.signatures.size < MAX_POLLING_SIGNATURES; + if (isNewFailure) this.pollingOutage.signatures.add(signature); + + if (isNewFailure || now - this.pollingOutage.lastReportAt >= POLLING_REPORT_INTERVAL_MS) { + this.pollingOutage.lastReportAt = now; + this.logger.warn(`Telegram polling failing (attempt ${this.pollingOutage.attempts}): ${message}`); + } + + if (this.pollingRecoveryTimer) clearTimeout(this.pollingRecoveryTimer); + this.pollingRecoveryTimer = setTimeout(() => this.onPollingRecovered(), POLLING_RECOVERY_GRACE_MS); + this.pollingRecoveryTimer.unref(); + } + + private onPollingRecovered(): void { + if (!this.pollingOutage) return; + + const { since, lastAt, attempts } = this.pollingOutage; + this.pollingOutage = null; + this.pollingRecoveryTimer = null; + + // A single failure needs no closing line. Silence only proves that no further error + // arrived — the poll itself is not probed here, so the line does not claim more. + if (attempts <= 1) return; + + this.logger.warn(`Telegram polling errors stopped after ${Math.round((lastAt - since) / 1000)}s and ${attempts} attempts`); + } + private async doSendMessage(group: string | number, message: string): Promise { await this.bot.sendMessage(group.toString(), message, { parse_mode: 'Markdown', disable_web_page_preview: true }); } diff --git a/socialmedia/telegram/telegram.types.ts b/socialmedia/telegram/telegram.types.ts index 814f058..947806e 100644 --- a/socialmedia/telegram/telegram.types.ts +++ b/socialmedia/telegram/telegram.types.ts @@ -16,6 +16,16 @@ export type TelegramSubscriptionState = { tradeUpdates: number; }; +// @dev: in-memory state of an ongoing telegram polling outage, used to report it once +// per distinct failure instead of once per retry +export type TelegramPollingOutage = { + since: number; + lastAt: number; + lastReportAt: number; + attempts: number; + signatures: Set; +}; + export type TelegramGroupState = { apiVersion: string; createdAt: number; diff --git a/socialmedia/twitter/twitter.service.ts b/socialmedia/twitter/twitter.service.ts index b86c21b..b68a967 100644 --- a/socialmedia/twitter/twitter.service.ts +++ b/socialmedia/twitter/twitter.service.ts @@ -3,6 +3,7 @@ import { CONFIG } from 'api.config'; import { StablecoinBridgeQuery } from 'bridge/bridge.types'; import { EcosystemMintQueryItem } from 'ecosystem/ecosystem.stablecoin.types'; import { FrontendCodeRegisteredQuery, FrontendCodeSavingsQuery } from 'frontendcode/frontendcode.types'; +import { resolveMediaPath } from 'socialmedia/socialmedia.helper'; import { SocialMediaFct, SocialMediaService } from 'socialmedia/socialmedia.service'; import { TradeQuery } from 'trades/trade.types'; import { SendTweetV2Params, TwitterApi } from 'twitter-api-v2'; @@ -16,6 +17,7 @@ import { TradeMessage } from './messages/Trade.message'; export class TwitterService implements OnModuleInit, SocialMediaFct { private readonly logger = new Logger(this.constructor.name); private readonly client: TwitterApi | null; + private readonly reportedMissingMedia = new Set(); constructor(private readonly socialMediaService: SocialMediaService) { this.client = CONFIG.twitter @@ -76,8 +78,16 @@ export class TwitterService implements OnModuleInit, SocialMediaFct { text: message, }; - if (media) { - const mediaId = await this.client.v1.uploadMedia(media).catch((e) => this.logger.error('uploadMedia failed', e)); + // Same best-effort handling as the telegram path: a missing asset must not turn + // every post into an upload error, the post simply goes out without media. + const mediaPath = resolveMediaPath(media); + if (media && !mediaPath && !this.reportedMissingMedia.has(media)) { + this.reportedMissingMedia.add(media); + this.logger.warn(`Twitter media asset missing: ${media} — posting text-only`); + } + + if (mediaPath) { + const mediaId = await this.client.v1.uploadMedia(mediaPath).catch((e) => this.logger.error('uploadMedia failed', e)); if (mediaId) tweetParams.media = { media_ids: [mediaId] }; }