From 6edd5aa95a7cfaf4782a84873f5c7e670067e6e8 Mon Sep 17 00:00:00 2001 From: Daniel Padrino Date: Wed, 29 Jul 2026 12:33:17 -0300 Subject: [PATCH 1/3] Fall back to text-only posts and collapse Telegram polling noise MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TELEGRAM_IMAGES_DIR is unset in the deployed environment, so every video-bearing notification builds the literal path "undefined/.mp4". node-telegram-bot-api reinterprets a path that is not a readable file as a URL, Telegram answers "400 Bad Request: wrong HTTP URL specified", and sendMessage's catch only logs a warning — the notification is dropped. Trade, MintingUpdate, SavingUpdate, StablecoinBridgeUpdate and FrontendCodeRegistered have therefore never reached subscribers, while text-only alerts kept working and made the bot look healthy. Setting the env var alone would not help: the media files are not in the image. Port the guard d-EURO/api#115 already uses: resolveMediaPath checks the file with existsSync and returns undefined, so sendMessage degrades to sendMessage instead of sendVideo. Separately, attach a 'polling_error' listener. Without one the library writes its own unformatted console error for every failed poll, and it retries on a fixed interval with no backoff — a nine-second Telegram gateway outage produced 27 error lines, and sustained retries earn a 429 on top of the original 502. The listener reports one line per distinct failure plus one on recovery with duration and attempt count, which turns the same outage into two lines while still surfacing an escalation from 502 to 429. Both lines are logged at warn so an outage never appears to stay open under a warn-level log configuration. No unit tests added: the repo's jest config has rootDir="src" but the sources live at the repo root, so `yarn test` finds zero tests today. --- socialmedia/socialmedia.helper.ts | 8 ++++ socialmedia/telegram/telegram.service.ts | 52 +++++++++++++++++++++++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/socialmedia/socialmedia.helper.ts b/socialmedia/socialmedia.helper.ts index 9cfc532..cbfe6a3 100644 --- a/socialmedia/socialmedia.helper.ts +++ b/socialmedia/socialmedia.helper.ts @@ -1,7 +1,15 @@ +import { existsSync } from 'fs'; import { Hex, hexToString } from 'viem'; const JUICE_WALLET_FRONTEND_CODE = '0xe8d44050873dba865aa7c170ab4cce64d90839a34dcfd6cf71d14e0205443b1b'; +// Returns the input path when the asset is readable on disk, otherwise undefined. +// Missing notification assets must not block posting — callers fall back to text-only. +export function resolveMediaPath(media: string | undefined): string | undefined { + if (!media) return undefined; + return existsSync(media) ? media : undefined; +} + export function createRefCode(frontendCode: string): string | undefined { if (frontendCode?.startsWith('0x00')) { return hexToString(frontendCode as Hex).replace(/[\x00-\x1f,\x7f]/g, ''); diff --git a/socialmedia/telegram/telegram.service.ts b/socialmedia/telegram/telegram.service.ts index 915625e..bc7a8ac 100644 --- a/socialmedia/telegram/telegram.service.ts +++ b/socialmedia/telegram/telegram.service.ts @@ -8,6 +8,7 @@ import { FrontendCodeRegisteredQuery, FrontendCodeSavingsQuery } from 'frontendc import TelegramBot from 'node-telegram-bot-api'; import { PositionsService } from 'positions/positions.service'; import { SavingsLeadrateService } from 'savings/savings.leadrate.service'; +import { resolveMediaPath } from 'socialmedia/socialmedia.helper'; import { SocialMediaFct, SocialMediaService } from 'socialmedia/socialmedia.service'; import { StorageService } from 'storage/storage.service'; import { TradeQuery } from 'trades/trade.types'; @@ -27,6 +28,9 @@ import { StablecoinBridgeMessage } from './messages/StablecoinBridgeUpdate.messa import { TradeMessage } from './messages/Trade.message'; import { TelegramGroupState, TelegramState } from './telegram.types'; +// Consider polling healthy again once no further error arrived for this long. +const POLLING_RECOVERY_GRACE_MS = 30_000; + @Injectable() export class TelegramService implements OnModuleInit, SocialMediaFct { private readonly logger = new Logger(this.constructor.name); @@ -34,6 +38,8 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { private readonly telegramHandles: string[] = ['/start', '/subscribe', '/unsubscribe', '/help']; private readonly telegramState: TelegramState; private telegramGroupState: TelegramGroupState; + private pollingOutage: { since: number; lastAt: number; attempts: number; signature: string } | null = null; + private pollingRecoveryTimer: NodeJS.Timeout | null = null; constructor( private readonly socialMediaService: SocialMediaService, @@ -44,6 +50,9 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { private readonly challenge: ChallengesService ) { this.bot = CONFIG.telegram ? new TelegramBot(CONFIG.telegram.botToken, { polling: true }) : null; + // Without a 'polling_error' listener node-telegram-bot-api writes its own unformatted + // console error for every failed poll, and it retries on a fixed interval with no backoff. + this.bot?.on('polling_error', (error) => this.onPollingError(error)); const time: number = Date.now() + 365 * 24 * 60 * 60 * 1000; @@ -241,7 +250,14 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { private async sendMessage(group: string | number, message: string, video?: string) { try { this.logger.log(`Sending message to group id: ${group}`); - video ? await this.doSendVideo(group, message, video) : await this.doSendMessage(group, message); + // Notification assets are best-effort — when the file is missing fall back to a + // text-only message rather than letting node-telegram-bot-api reinterpret a missing + // local path as a URL (which produces "wrong HTTP URL specified" 400s on every send). + const videoPath = resolveMediaPath(video); + if (video && !videoPath) { + this.logger.debug(`Telegram video asset missing: ${video} — sending text-only`); + } + videoPath ? await this.doSendVideo(group, message, videoPath) : await this.doSendMessage(group, message); } catch (error) { const msg = { notFound: 'chat not found', @@ -268,6 +284,40 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { } } + // Telegram's gateway answers 502/504 during its own restarts, so a short outage would + // otherwise produce several identical lines per second. Report one line per distinct + // failure and one on recovery, both at warn so an outage never appears to stay open. + private onPollingError(error: Error & { code?: string }): void { + const signature = error?.message ?? String(error); + const now = Date.now(); + + if (!this.pollingOutage) { + this.pollingOutage = { since: now, lastAt: now, attempts: 0, signature: '' }; + } + + this.pollingOutage.attempts++; + this.pollingOutage.lastAt = now; + + if (this.pollingOutage.signature !== signature) { + this.pollingOutage.signature = signature; + this.logger.warn(`Telegram polling failing: ${signature}`); + } + + if (this.pollingRecoveryTimer) clearTimeout(this.pollingRecoveryTimer); + this.pollingRecoveryTimer = setTimeout(() => this.onPollingRecovered(), POLLING_RECOVERY_GRACE_MS); + this.pollingRecoveryTimer.unref(); + } + + private onPollingRecovered(): void { + if (!this.pollingOutage) return; + + const { since, lastAt, attempts } = this.pollingOutage; + this.pollingOutage = null; + this.pollingRecoveryTimer = null; + + this.logger.warn(`Telegram polling recovered after ${Math.round((lastAt - since) / 1000)}s and ${attempts} failed attempts`); + } + private async doSendMessage(group: string | number, message: string): Promise { await this.bot.sendMessage(group.toString(), message, { parse_mode: 'Markdown', disable_web_page_preview: true }); } From 161bb0970ac6c3169181deeeddd541758b7154cd Mon Sep 17 00:00:00 2001 From: Daniel Padrino Date: Wed, 29 Jul 2026 12:45:59 -0300 Subject: [PATCH 2/3] Keep missing assets and unrecovered outages visible in the logs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up on the first two commits. The missing-asset notice was logged at debug, which the deployed logger never emits. Since the assets are absent from the image the fallback is the permanent state, so the condition would have become completely silent — it was visible as a warning before this branch. Report it at warn, once per distinct asset per boot. The polling signature used the raw error message compared against the previous one only, which fails in the two cases that actually occur: "Too Many Requests: retry after N" counts down and a connect failure carries a rotating gateway address, so each retry looked like a new failure, and alternating errors re-reported on every poll. Mask digits out of the signature and track the signatures already reported within the current outage. A permanently failing poll — a revoked token answers 401 forever — reported once and then stayed silent, because the recovery timer only fires once errors stop. Repeat the report every five minutes while the outage is open. Also apply the same media guard to TwitterService, which is the other caller of the shared helper and logged an upload error per post, move the outage shape into telegram.types.ts next to the other state types, and truncate the error text, which wraps the upstream response body. --- socialmedia/telegram/telegram.service.ts | 33 +++++++++++++++++------- socialmedia/telegram/telegram.types.ts | 10 +++++++ socialmedia/twitter/twitter.service.ts | 14 ++++++++-- 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/socialmedia/telegram/telegram.service.ts b/socialmedia/telegram/telegram.service.ts index bc7a8ac..f5c3e00 100644 --- a/socialmedia/telegram/telegram.service.ts +++ b/socialmedia/telegram/telegram.service.ts @@ -26,10 +26,14 @@ import { PositionProposalMessage } from './messages/PositionProposal.message'; import { SavingUpdateMessage } from './messages/SavingUpdate.message'; import { StablecoinBridgeMessage } from './messages/StablecoinBridgeUpdate.message'; import { TradeMessage } from './messages/Trade.message'; -import { TelegramGroupState, TelegramState } from './telegram.types'; +import { TelegramGroupState, TelegramPollingOutage, TelegramState } from './telegram.types'; // Consider polling healthy again once no further error arrived for this long. const POLLING_RECOVERY_GRACE_MS = 30_000; +// Repeat an ongoing outage at this interval so a permanent failure never goes silent. +const POLLING_REPORT_INTERVAL_MS = 300_000; +// Telegram errors wrap the upstream response body, which is not always short. +const MAX_POLLING_ERROR_LENGTH = 200; @Injectable() export class TelegramService implements OnModuleInit, SocialMediaFct { @@ -38,8 +42,9 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { private readonly telegramHandles: string[] = ['/start', '/subscribe', '/unsubscribe', '/help']; private readonly telegramState: TelegramState; private telegramGroupState: TelegramGroupState; - private pollingOutage: { since: number; lastAt: number; attempts: number; signature: string } | null = null; + private pollingOutage: TelegramPollingOutage | null = null; private pollingRecoveryTimer: NodeJS.Timeout | null = null; + private readonly reportedMissingMedia = new Set(); constructor( private readonly socialMediaService: SocialMediaService, @@ -254,8 +259,11 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { // text-only message rather than letting node-telegram-bot-api reinterpret a missing // local path as a URL (which produces "wrong HTTP URL specified" 400s on every send). const videoPath = resolveMediaPath(video); - if (video && !videoPath) { - this.logger.debug(`Telegram video asset missing: ${video} — sending text-only`); + if (video && !videoPath && !this.reportedMissingMedia.has(video)) { + // Once per asset per boot: a missing asset is a deployment defect that never + // heals on its own, so it must stay visible without repeating on every send. + this.reportedMissingMedia.add(video); + this.logger.warn(`Telegram video asset missing: ${video} — sending text-only`); } videoPath ? await this.doSendVideo(group, message, videoPath) : await this.doSendMessage(group, message); } catch (error) { @@ -288,19 +296,26 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { // otherwise produce several identical lines per second. Report one line per distinct // failure and one on recovery, both at warn so an outage never appears to stay open. private onPollingError(error: Error & { code?: string }): void { - const signature = error?.message ?? String(error); + const message = (error?.message ?? String(error)).slice(0, MAX_POLLING_ERROR_LENGTH); + // Digits carry the volatile parts of a telegram error — the countdown in + // "retry after 5" and the rotating gateway address in a connect failure — so they are + // masked out of the signature to keep one repeating failure on one line. + const signature = `${error?.code ?? 'UNKNOWN'}: ${message.replace(/\d+/g, '#')}`; const now = Date.now(); if (!this.pollingOutage) { - this.pollingOutage = { since: now, lastAt: now, attempts: 0, signature: '' }; + this.pollingOutage = { since: now, lastAt: now, lastReportAt: 0, attempts: 0, signatures: new Set() }; } this.pollingOutage.attempts++; this.pollingOutage.lastAt = now; - if (this.pollingOutage.signature !== signature) { - this.pollingOutage.signature = signature; - this.logger.warn(`Telegram polling failing: ${signature}`); + const isNewFailure = !this.pollingOutage.signatures.has(signature); + if (isNewFailure) this.pollingOutage.signatures.add(signature); + + if (isNewFailure || now - this.pollingOutage.lastReportAt >= POLLING_REPORT_INTERVAL_MS) { + this.pollingOutage.lastReportAt = now; + this.logger.warn(`Telegram polling failing (attempt ${this.pollingOutage.attempts}): ${message}`); } if (this.pollingRecoveryTimer) clearTimeout(this.pollingRecoveryTimer); diff --git a/socialmedia/telegram/telegram.types.ts b/socialmedia/telegram/telegram.types.ts index 814f058..947806e 100644 --- a/socialmedia/telegram/telegram.types.ts +++ b/socialmedia/telegram/telegram.types.ts @@ -16,6 +16,16 @@ export type TelegramSubscriptionState = { tradeUpdates: number; }; +// @dev: in-memory state of an ongoing telegram polling outage, used to report it once +// per distinct failure instead of once per retry +export type TelegramPollingOutage = { + since: number; + lastAt: number; + lastReportAt: number; + attempts: number; + signatures: Set; +}; + export type TelegramGroupState = { apiVersion: string; createdAt: number; diff --git a/socialmedia/twitter/twitter.service.ts b/socialmedia/twitter/twitter.service.ts index b86c21b..b68a967 100644 --- a/socialmedia/twitter/twitter.service.ts +++ b/socialmedia/twitter/twitter.service.ts @@ -3,6 +3,7 @@ import { CONFIG } from 'api.config'; import { StablecoinBridgeQuery } from 'bridge/bridge.types'; import { EcosystemMintQueryItem } from 'ecosystem/ecosystem.stablecoin.types'; import { FrontendCodeRegisteredQuery, FrontendCodeSavingsQuery } from 'frontendcode/frontendcode.types'; +import { resolveMediaPath } from 'socialmedia/socialmedia.helper'; import { SocialMediaFct, SocialMediaService } from 'socialmedia/socialmedia.service'; import { TradeQuery } from 'trades/trade.types'; import { SendTweetV2Params, TwitterApi } from 'twitter-api-v2'; @@ -16,6 +17,7 @@ import { TradeMessage } from './messages/Trade.message'; export class TwitterService implements OnModuleInit, SocialMediaFct { private readonly logger = new Logger(this.constructor.name); private readonly client: TwitterApi | null; + private readonly reportedMissingMedia = new Set(); constructor(private readonly socialMediaService: SocialMediaService) { this.client = CONFIG.twitter @@ -76,8 +78,16 @@ export class TwitterService implements OnModuleInit, SocialMediaFct { text: message, }; - if (media) { - const mediaId = await this.client.v1.uploadMedia(media).catch((e) => this.logger.error('uploadMedia failed', e)); + // Same best-effort handling as the telegram path: a missing asset must not turn + // every post into an upload error, the post simply goes out without media. + const mediaPath = resolveMediaPath(media); + if (media && !mediaPath && !this.reportedMissingMedia.has(media)) { + this.reportedMissingMedia.add(media); + this.logger.warn(`Twitter media asset missing: ${media} — posting text-only`); + } + + if (mediaPath) { + const mediaId = await this.client.v1.uploadMedia(mediaPath).catch((e) => this.logger.error('uploadMedia failed', e)); if (mediaId) tweetParams.media = { media_ids: [mediaId] }; } From 6c8d3debf1c09617228ce91a0d23a624e8c6f700 Mon Sep 17 00:00:00 2001 From: Daniel Padrino Date: Wed, 29 Jul 2026 12:57:25 -0300 Subject: [PATCH 3/3] Bound the outage signature set and stop over-claiming recovery Second review follow-up. The signature prefixed the error code onto a message that the library already prefixes with that same code, so the prefix discriminated nothing. Use the message alone. The signature set had no upper bound: a message carrying a per-attempt token that survives digit masking - a request id in an upstream error body reaches the parse-error branch verbatim - produced a fresh signature per poll, which defeated the collapsing and grew the set without limit. Cap it, past which only the periodic report remains. A replay of 2000 such polls now yields 22 lines and 20 retained signatures instead of 2000 of each. An error arriving more than the grace period after the previous one opened and closed its own outage, so an isolated blip cost two lines where it used to cost one. Skip the closing line for a single attempt. The closing line said "recovered", but nothing probes the poll - the grace period only establishes that no further error arrived, and a request that never settles would look the same. Say what is actually known instead. --- socialmedia/telegram/telegram.service.ts | 26 ++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/socialmedia/telegram/telegram.service.ts b/socialmedia/telegram/telegram.service.ts index f5c3e00..5e22979 100644 --- a/socialmedia/telegram/telegram.service.ts +++ b/socialmedia/telegram/telegram.service.ts @@ -34,6 +34,8 @@ const POLLING_RECOVERY_GRACE_MS = 30_000; const POLLING_REPORT_INTERVAL_MS = 300_000; // Telegram errors wrap the upstream response body, which is not always short. const MAX_POLLING_ERROR_LENGTH = 200; +// Beyond this many distinct failures one outage reports on its interval only. +const MAX_POLLING_SIGNATURES = 20; @Injectable() export class TelegramService implements OnModuleInit, SocialMediaFct { @@ -294,13 +296,15 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { // Telegram's gateway answers 502/504 during its own restarts, so a short outage would // otherwise produce several identical lines per second. Report one line per distinct - // failure and one on recovery, both at warn so an outage never appears to stay open. - private onPollingError(error: Error & { code?: string }): void { + // failure, repeat an outage that never clears every POLLING_REPORT_INTERVAL_MS, and close + // it with one line — all at warn, so an outage never appears to stay open. + private onPollingError(error: Error): void { const message = (error?.message ?? String(error)).slice(0, MAX_POLLING_ERROR_LENGTH); - // Digits carry the volatile parts of a telegram error — the countdown in - // "retry after 5" and the rotating gateway address in a connect failure — so they are - // masked out of the signature to keep one repeating failure on one line. - const signature = `${error?.code ?? 'UNKNOWN'}: ${message.replace(/\d+/g, '#')}`; + // The library prefixes its own error code onto the message, so the message alone + // identifies the failure. Digits carry the volatile parts — the countdown in + // "retry after 5", the rotating gateway address in a connect failure — and are masked + // out to keep a repeating failure on one line. + const signature = message.replace(/\d+/g, '#'); const now = Date.now(); if (!this.pollingOutage) { @@ -310,7 +314,9 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { this.pollingOutage.attempts++; this.pollingOutage.lastAt = now; - const isNewFailure = !this.pollingOutage.signatures.has(signature); + // Past the cap only the periodic report remains, which bounds the log volume and the + // set itself if a message carries a token that survives digit masking. + const isNewFailure = !this.pollingOutage.signatures.has(signature) && this.pollingOutage.signatures.size < MAX_POLLING_SIGNATURES; if (isNewFailure) this.pollingOutage.signatures.add(signature); if (isNewFailure || now - this.pollingOutage.lastReportAt >= POLLING_REPORT_INTERVAL_MS) { @@ -330,7 +336,11 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { this.pollingOutage = null; this.pollingRecoveryTimer = null; - this.logger.warn(`Telegram polling recovered after ${Math.round((lastAt - since) / 1000)}s and ${attempts} failed attempts`); + // A single failure needs no closing line. Silence only proves that no further error + // arrived — the poll itself is not probed here, so the line does not claim more. + if (attempts <= 1) return; + + this.logger.warn(`Telegram polling errors stopped after ${Math.round((lastAt - since) / 1000)}s and ${attempts} attempts`); } private async doSendMessage(group: string | number, message: string): Promise {