diff --git a/socialmedia/socialmedia.helper.ts b/socialmedia/socialmedia.helper.ts index 9cfc532..cbfe6a3 100644 --- a/socialmedia/socialmedia.helper.ts +++ b/socialmedia/socialmedia.helper.ts @@ -1,7 +1,15 @@ +import { existsSync } from 'fs'; import { Hex, hexToString } from 'viem'; const JUICE_WALLET_FRONTEND_CODE = '0xe8d44050873dba865aa7c170ab4cce64d90839a34dcfd6cf71d14e0205443b1b'; +// Returns the input path when the asset is readable on disk, otherwise undefined. +// Missing notification assets must not block posting — callers fall back to text-only. +export function resolveMediaPath(media: string | undefined): string | undefined { + if (!media) return undefined; + return existsSync(media) ? media : undefined; +} + export function createRefCode(frontendCode: string): string | undefined { if (frontendCode?.startsWith('0x00')) { return hexToString(frontendCode as Hex).replace(/[\x00-\x1f,\x7f]/g, ''); diff --git a/socialmedia/telegram/telegram.service.ts b/socialmedia/telegram/telegram.service.ts index 915625e..5e22979 100644 --- a/socialmedia/telegram/telegram.service.ts +++ b/socialmedia/telegram/telegram.service.ts @@ -8,6 +8,7 @@ import { FrontendCodeRegisteredQuery, FrontendCodeSavingsQuery } from 'frontendc import TelegramBot from 'node-telegram-bot-api'; import { PositionsService } from 'positions/positions.service'; import { SavingsLeadrateService } from 'savings/savings.leadrate.service'; +import { resolveMediaPath } from 'socialmedia/socialmedia.helper'; import { SocialMediaFct, SocialMediaService } from 'socialmedia/socialmedia.service'; import { StorageService } from 'storage/storage.service'; import { TradeQuery } from 'trades/trade.types'; @@ -25,7 +26,16 @@ import { PositionProposalMessage } from './messages/PositionProposal.message'; import { SavingUpdateMessage } from './messages/SavingUpdate.message'; import { StablecoinBridgeMessage } from './messages/StablecoinBridgeUpdate.message'; import { TradeMessage } from './messages/Trade.message'; -import { TelegramGroupState, TelegramState } from './telegram.types'; +import { TelegramGroupState, TelegramPollingOutage, TelegramState } from './telegram.types'; + +// Consider polling healthy again once no further error arrived for this long. +const POLLING_RECOVERY_GRACE_MS = 30_000; +// Repeat an ongoing outage at this interval so a permanent failure never goes silent. +const POLLING_REPORT_INTERVAL_MS = 300_000; +// Telegram errors wrap the upstream response body, which is not always short. +const MAX_POLLING_ERROR_LENGTH = 200; +// Beyond this many distinct failures one outage reports on its interval only. +const MAX_POLLING_SIGNATURES = 20; @Injectable() export class TelegramService implements OnModuleInit, SocialMediaFct { @@ -34,6 +44,9 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { private readonly telegramHandles: string[] = ['/start', '/subscribe', '/unsubscribe', '/help']; private readonly telegramState: TelegramState; private telegramGroupState: TelegramGroupState; + private pollingOutage: TelegramPollingOutage | null = null; + private pollingRecoveryTimer: NodeJS.Timeout | null = null; + private readonly reportedMissingMedia = new Set(); constructor( private readonly socialMediaService: SocialMediaService, @@ -44,6 +57,9 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { private readonly challenge: ChallengesService ) { this.bot = CONFIG.telegram ? new TelegramBot(CONFIG.telegram.botToken, { polling: true }) : null; + // Without a 'polling_error' listener node-telegram-bot-api writes its own unformatted + // console error for every failed poll, and it retries on a fixed interval with no backoff. + this.bot?.on('polling_error', (error) => this.onPollingError(error)); const time: number = Date.now() + 365 * 24 * 60 * 60 * 1000; @@ -241,7 +257,17 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { private async sendMessage(group: string | number, message: string, video?: string) { try { this.logger.log(`Sending message to group id: ${group}`); - video ? await this.doSendVideo(group, message, video) : await this.doSendMessage(group, message); + // Notification assets are best-effort — when the file is missing fall back to a + // text-only message rather than letting node-telegram-bot-api reinterpret a missing + // local path as a URL (which produces "wrong HTTP URL specified" 400s on every send). + const videoPath = resolveMediaPath(video); + if (video && !videoPath && !this.reportedMissingMedia.has(video)) { + // Once per asset per boot: a missing asset is a deployment defect that never + // heals on its own, so it must stay visible without repeating on every send. + this.reportedMissingMedia.add(video); + this.logger.warn(`Telegram video asset missing: ${video} — sending text-only`); + } + videoPath ? await this.doSendVideo(group, message, videoPath) : await this.doSendMessage(group, message); } catch (error) { const msg = { notFound: 'chat not found', @@ -268,6 +294,55 @@ export class TelegramService implements OnModuleInit, SocialMediaFct { } } + // Telegram's gateway answers 502/504 during its own restarts, so a short outage would + // otherwise produce several identical lines per second. Report one line per distinct + // failure, repeat an outage that never clears every POLLING_REPORT_INTERVAL_MS, and close + // it with one line — all at warn, so an outage never appears to stay open. + private onPollingError(error: Error): void { + const message = (error?.message ?? String(error)).slice(0, MAX_POLLING_ERROR_LENGTH); + // The library prefixes its own error code onto the message, so the message alone + // identifies the failure. Digits carry the volatile parts — the countdown in + // "retry after 5", the rotating gateway address in a connect failure — and are masked + // out to keep a repeating failure on one line. + const signature = message.replace(/\d+/g, '#'); + const now = Date.now(); + + if (!this.pollingOutage) { + this.pollingOutage = { since: now, lastAt: now, lastReportAt: 0, attempts: 0, signatures: new Set() }; + } + + this.pollingOutage.attempts++; + this.pollingOutage.lastAt = now; + + // Past the cap only the periodic report remains, which bounds the log volume and the + // set itself if a message carries a token that survives digit masking. + const isNewFailure = !this.pollingOutage.signatures.has(signature) && this.pollingOutage.signatures.size < MAX_POLLING_SIGNATURES; + if (isNewFailure) this.pollingOutage.signatures.add(signature); + + if (isNewFailure || now - this.pollingOutage.lastReportAt >= POLLING_REPORT_INTERVAL_MS) { + this.pollingOutage.lastReportAt = now; + this.logger.warn(`Telegram polling failing (attempt ${this.pollingOutage.attempts}): ${message}`); + } + + if (this.pollingRecoveryTimer) clearTimeout(this.pollingRecoveryTimer); + this.pollingRecoveryTimer = setTimeout(() => this.onPollingRecovered(), POLLING_RECOVERY_GRACE_MS); + this.pollingRecoveryTimer.unref(); + } + + private onPollingRecovered(): void { + if (!this.pollingOutage) return; + + const { since, lastAt, attempts } = this.pollingOutage; + this.pollingOutage = null; + this.pollingRecoveryTimer = null; + + // A single failure needs no closing line. Silence only proves that no further error + // arrived — the poll itself is not probed here, so the line does not claim more. + if (attempts <= 1) return; + + this.logger.warn(`Telegram polling errors stopped after ${Math.round((lastAt - since) / 1000)}s and ${attempts} attempts`); + } + private async doSendMessage(group: string | number, message: string): Promise { await this.bot.sendMessage(group.toString(), message, { parse_mode: 'Markdown', disable_web_page_preview: true }); } diff --git a/socialmedia/telegram/telegram.types.ts b/socialmedia/telegram/telegram.types.ts index 814f058..947806e 100644 --- a/socialmedia/telegram/telegram.types.ts +++ b/socialmedia/telegram/telegram.types.ts @@ -16,6 +16,16 @@ export type TelegramSubscriptionState = { tradeUpdates: number; }; +// @dev: in-memory state of an ongoing telegram polling outage, used to report it once +// per distinct failure instead of once per retry +export type TelegramPollingOutage = { + since: number; + lastAt: number; + lastReportAt: number; + attempts: number; + signatures: Set; +}; + export type TelegramGroupState = { apiVersion: string; createdAt: number; diff --git a/socialmedia/twitter/twitter.service.ts b/socialmedia/twitter/twitter.service.ts index b86c21b..b68a967 100644 --- a/socialmedia/twitter/twitter.service.ts +++ b/socialmedia/twitter/twitter.service.ts @@ -3,6 +3,7 @@ import { CONFIG } from 'api.config'; import { StablecoinBridgeQuery } from 'bridge/bridge.types'; import { EcosystemMintQueryItem } from 'ecosystem/ecosystem.stablecoin.types'; import { FrontendCodeRegisteredQuery, FrontendCodeSavingsQuery } from 'frontendcode/frontendcode.types'; +import { resolveMediaPath } from 'socialmedia/socialmedia.helper'; import { SocialMediaFct, SocialMediaService } from 'socialmedia/socialmedia.service'; import { TradeQuery } from 'trades/trade.types'; import { SendTweetV2Params, TwitterApi } from 'twitter-api-v2'; @@ -16,6 +17,7 @@ import { TradeMessage } from './messages/Trade.message'; export class TwitterService implements OnModuleInit, SocialMediaFct { private readonly logger = new Logger(this.constructor.name); private readonly client: TwitterApi | null; + private readonly reportedMissingMedia = new Set(); constructor(private readonly socialMediaService: SocialMediaService) { this.client = CONFIG.twitter @@ -76,8 +78,16 @@ export class TwitterService implements OnModuleInit, SocialMediaFct { text: message, }; - if (media) { - const mediaId = await this.client.v1.uploadMedia(media).catch((e) => this.logger.error('uploadMedia failed', e)); + // Same best-effort handling as the telegram path: a missing asset must not turn + // every post into an upload error, the post simply goes out without media. + const mediaPath = resolveMediaPath(media); + if (media && !mediaPath && !this.reportedMissingMedia.has(media)) { + this.reportedMissingMedia.add(media); + this.logger.warn(`Twitter media asset missing: ${media} — posting text-only`); + } + + if (mediaPath) { + const mediaId = await this.client.v1.uploadMedia(mediaPath).catch((e) => this.logger.error('uploadMedia failed', e)); if (mediaId) tweetParams.media = { media_ids: [mediaId] }; }