From bf34c4bd8fd4160e14b55a76e41e43d5b6343c7f Mon Sep 17 00:00:00 2001 From: Niladri Adhikary Date: Thu, 30 Jul 2026 11:54:35 +0530 Subject: [PATCH 1/4] feat(alerts): add re-notify interval, stop notifying on every evaluation Signed-off-by: Niladri Adhikary --- .changeset/alert-renotify-interval.md | 7 + packages/api/openapi.json | 7 + packages/api/src/controllers/alerts.ts | 3 + packages/api/src/models/alert.ts | 8 + packages/api/src/models/alertHistory.ts | 5 + packages/api/src/routers/api/alerts.ts | 1 + .../api/src/routers/external-api/v2/alerts.ts | 6 + .../__tests__/checkAlerts.int.test.ts | 199 +++++++++++++++++- packages/api/src/tasks/checkAlerts/index.ts | 93 ++++++-- packages/api/src/utils/externalApi.ts | 2 + packages/api/src/utils/zod.ts | 1 + packages/app/src/DBSearchPageAlertModal.tsx | 9 + .../src/components/AlertScheduleFields.tsx | 47 ++++- .../DBEditTimeChartForm/TileAlertEditor.tsx | 6 + packages/common-utils/src/types.ts | 2 + 15 files changed, 369 insertions(+), 27 deletions(-) create mode 100644 .changeset/alert-renotify-interval.md diff --git a/.changeset/alert-renotify-interval.md b/.changeset/alert-renotify-interval.md new file mode 100644 index 0000000000..b8ddc6e1bd --- /dev/null +++ b/.changeset/alert-renotify-interval.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/common-utils': minor +'@hyperdx/app': minor +'@hyperdx/api': minor +--- + +feat: alerts now notify once on the OK -> ALERT transition instead of on every evaluation while firing. A sustained breach used to send a duplicate notification on every evaluation interval; a new `renotifyIntervalMinutes` setting ("Re-notify every" under Advanced Settings) controls repeats instead — leave it unset for transition-only (the new default), set `0` to restore the old notify-on-every-evaluation behavior, or set N to re-notify every N minutes. Resolve notifications are unchanged. This is a behavior change for existing alerts: set `renotifyIntervalMinutes: 0` to keep the old behavior. diff --git a/packages/api/openapi.json b/packages/api/openapi.json index 4c076f92fa..11a0e2b7c5 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -300,6 +300,13 @@ "nullable": true, "description": "Fire the alert only after its condition has been met for this many consecutive evaluation windows. While the condition is met but fewer than this many consecutive windows have violated, the alert is in the PENDING state.", "example": 3 + }, + "renotifyIntervalMinutes": { + "type": "integer", + "minimum": 0, + "nullable": true, + "description": "How often to repeat the notification while the alert keeps firing. Omit or set to null to notify only once when the alert transitions from OK to ALERT, 0 to notify on every evaluation, or a positive number to re-notify at most once every that many minutes. A resolve notification is always sent when the alert returns to OK.", + "example": 60 } } }, diff --git a/packages/api/src/controllers/alerts.ts b/packages/api/src/controllers/alerts.ts index ff430657c9..87cb7cad99 100644 --- a/packages/api/src/controllers/alerts.ts +++ b/packages/api/src/controllers/alerts.ts @@ -158,6 +158,9 @@ const makeAlert = (alert: AlertInput, userId?: ObjectId): Partial => { // Multi-window alerting numConsecutiveWindows: alert.numConsecutiveWindows ?? null, + + // Re-notification while firing + renotifyIntervalMinutes: alert.renotifyIntervalMinutes ?? null, }; }; diff --git a/packages/api/src/models/alert.ts b/packages/api/src/models/alert.ts index f84bc30020..ed1bd32574 100644 --- a/packages/api/src/models/alert.ts +++ b/packages/api/src/models/alert.ts @@ -88,6 +88,9 @@ export interface IAlert { // Multi-window alerting: fire only after N violations in M consecutive windows numConsecutiveWindows?: number | null; + // Re-notify while firing: null = transition only, 0 = every evaluation, N = every N minutes. + renotifyIntervalMinutes?: number | null; + // Errors recorded during the most recent execution executionErrors?: IAlertError[]; createdAt: Date; @@ -199,6 +202,11 @@ const AlertSchema = new Schema( required: false, min: 1, }, + renotifyIntervalMinutes: { + type: Number, + required: false, + min: 0, + }, silenced: { required: false, type: { diff --git a/packages/api/src/models/alertHistory.ts b/packages/api/src/models/alertHistory.ts index 5cf8630f81..97d29279e6 100644 --- a/packages/api/src/models/alertHistory.ts +++ b/packages/api/src/models/alertHistory.ts @@ -13,6 +13,7 @@ export interface IAlertHistory { lastValues: { startTime: Date; count: number }[]; group?: string; // For group-by alerts, stores the group identifier fired?: boolean; + lastNotifiedAt?: Date; } const AlertHistorySchema = new Schema({ @@ -50,6 +51,10 @@ const AlertHistorySchema = new Schema({ type: Boolean, required: false, }, + lastNotifiedAt: { + type: Date, + required: false, + }, }); AlertHistorySchema.index( diff --git a/packages/api/src/routers/api/alerts.ts b/packages/api/src/routers/api/alerts.ts index 1ffe35bbb4..d5da470c2d 100644 --- a/packages/api/src/routers/api/alerts.ts +++ b/packages/api/src/routers/api/alerts.ts @@ -87,6 +87,7 @@ const formatAlertResponse = ( 'updatedAt', 'executionErrors', 'numConsecutiveWindows', + 'renotifyIntervalMinutes', ]), }; }; diff --git a/packages/api/src/routers/external-api/v2/alerts.ts b/packages/api/src/routers/external-api/v2/alerts.ts index e53aedf4bc..38a5a8c996 100644 --- a/packages/api/src/routers/external-api/v2/alerts.ts +++ b/packages/api/src/routers/external-api/v2/alerts.ts @@ -198,6 +198,12 @@ import { alertSchema, objectIdSchema } from '@/utils/zod'; * nullable: true * description: Fire the alert only after its condition has been met for this many consecutive evaluation windows. While the condition is met but fewer than this many consecutive windows have violated, the alert is in the PENDING state. * example: 3 + * renotifyIntervalMinutes: + * type: integer + * minimum: 0 + * nullable: true + * description: How often to repeat the notification while the alert keeps firing. Omit or set to null to notify only once when the alert transitions from OK to ALERT, 0 to notify on every evaluation, or a positive number to re-notify at most once every that many minutes. A resolve notification is always sent when the alert returns to OK. + * example: 60 * * AlertResponse: * allOf: diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index 0e03d17cdb..b7c17873d3 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -2380,6 +2380,8 @@ describe('checkAlerts', () => { // check if webhook was triggered // We're only checking the general structure here since the exact text includes timestamps + // Transition + resolve only; the second ALERT window doesn't re-notify. + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(2); expect(slack.postMessageToWebhook).toHaveBeenNthCalledWith( 1, 'https://hooks.slack.com/services/123', @@ -2397,7 +2399,7 @@ describe('checkAlerts', () => { 2, 'https://hooks.slack.com/services/123', { - text: '🚨 Alert for "My Search" - 1 lines found', + text: '✅ Alert for "My Search" - 0 lines found', blocks: [ { text: expect.any(Object), @@ -5130,9 +5132,8 @@ describe('checkAlerts', () => { // Check webhook calls: // 1-2: First run alerts for service-a and service-b - // 3: Second run alert for service-a - // 4: Second run resolution notification for service-b - expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(4); + // 3: Second run resolution notification for service-b (service-a doesn't re-notify) + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(3); // Verify the resolution notification was sent for service-b const calls = (slack.postMessageToWebhook as jest.Mock).mock.calls; @@ -5606,9 +5607,8 @@ describe('checkAlerts', () => { // Check webhook calls: // 1-2: First run alerts for service-a and service-b - // 3: Second run alert for service-a (continues alerting) - // 4: Second run resolution notification for service-b - expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(4); + // 3: Second run resolution notification for service-b (service-a doesn't re-notify) + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(3); // Verify the resolution notification was sent for service-b const calls = (slack.postMessageToWebhook as jest.Mock).mock.calls; @@ -9096,6 +9096,191 @@ describe('checkAlerts', () => { expect(serviceBHistories[0].fired).toBeFalsy(); }); }); + + describe('re-notification (renotifyIntervalMinutes)', () => { + // A sustained breach across four consecutive 5m windows: every evaluation + // below sees data in its own window, so the alert stays in ALERT + // throughout without ever returning to OK. + const insertSustainedBreach = () => + bulkInsertLogs( + ['00:05:00', '00:10:00', '00:15:00', '00:20:00'].map(time => ({ + ServiceName: 'api', + Timestamp: new Date(`2024-01-01T${time}Z`), + SeverityText: 'error', + Body: 'err', + })), + ); + + const EVALUATION_TIMES = [ + '2024-01-01T00:12:00Z', + '2024-01-01T00:17:00Z', + '2024-01-01T00:22:00Z', + '2024-01-01T00:27:00Z', + ].map(t => new Date(t)); + + const runSustainedBreach = async ( + renotifyIntervalMinutes?: number | null, + ) => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + jest + .spyOn(slack, 'postMessageToWebhook') + .mockResolvedValue({ text: 'ok' }); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + // threshold=1 (not 0): ABOVE is inclusive, so threshold=0 would make + // an empty window breach too and the alert could never resolve. + threshold: 1, + savedSearchId: savedSearch.id, + renotifyIntervalMinutes, + }, + { taskType: AlertTaskType.SAVED_SEARCH, savedSearch }, + ); + + await insertSustainedBreach(); + + // Notification count observed after each successive evaluation. + const notificationCounts: number[] = []; + for (const now of EVALUATION_TIMES) { + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + notificationCounts.push( + jest.mocked(slack.postMessageToWebhook).mock.calls.length, + ); + } + + const histories = await AlertHistory.find({ + alert: details.alert.id, + }).sort({ createdAt: 1 }); + + return { notificationCounts, histories }; + }; + + it('notifies once on the OK->ALERT transition by default', async () => { + const { notificationCounts, histories } = await runSustainedBreach(); + + // One notification on the first breach, silence for the rest. + expect(notificationCounts).toEqual([1, 1, 1, 1]); + expect(histories.map(h => h.state)).toEqual([ + 'ALERT', + 'ALERT', + 'ALERT', + 'ALERT', + ]); + // lastNotifiedAt is pinned to the window that notified and carried + // forward unchanged across the silent evaluations. + expect(histories.map(h => h.lastNotifiedAt?.toISOString())).toEqual( + Array(4).fill('2024-01-01T00:10:00.000Z'), + ); + }); + + it('notifies on every evaluation when renotifyIntervalMinutes is 0', async () => { + const { notificationCounts, histories } = await runSustainedBreach(0); + + expect(notificationCounts).toEqual([1, 2, 3, 4]); + expect(histories.map(h => h.lastNotifiedAt?.toISOString())).toEqual([ + '2024-01-01T00:10:00.000Z', + '2024-01-01T00:15:00.000Z', + '2024-01-01T00:20:00.000Z', + '2024-01-01T00:25:00.000Z', + ]); + }); + + it('re-notifies only after renotifyIntervalMinutes has elapsed', async () => { + // 10m re-notification on a 5m interval: notify at 00:10, skip 00:15, + // re-notify at 00:20, skip 00:25. + const { notificationCounts, histories } = await runSustainedBreach(10); + + expect(notificationCounts).toEqual([1, 1, 2, 2]); + expect(histories.map(h => h.lastNotifiedAt?.toISOString())).toEqual([ + '2024-01-01T00:10:00.000Z', + '2024-01-01T00:10:00.000Z', + '2024-01-01T00:20:00.000Z', + '2024-01-01T00:20:00.000Z', + ]); + }); + + it('notifies again on a new OK->ALERT transition after resolving', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + jest + .spyOn(slack, 'postMessageToWebhook') + .mockResolvedValue({ text: 'ok' }); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { type: 'webhook', webhookId: webhook._id.toString() }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + savedSearchId: savedSearch.id, + }, + { taskType: AlertTaskType.SAVED_SEARCH, savedSearch }, + ); + + // Breach, then a quiet window, then breach again. + await bulkInsertLogs( + ['00:05:00', '00:15:00'].map(time => ({ + ServiceName: 'api', + Timestamp: new Date(`2024-01-01T${time}Z`), + SeverityText: 'error', + Body: 'err', + })), + ); + + for (const now of EVALUATION_TIMES.slice(0, 3)) { + await processAlertAtTime( + now, + details, + clickhouseClient, + connection, + alertProvider, + teamWebhooksById, + ); + } + + const histories = await AlertHistory.find({ + alert: details.alert.id, + }).sort({ createdAt: 1 }); + expect(histories.map(h => h.state)).toEqual(['ALERT', 'OK', 'ALERT']); + + // ALERT, resolved OK, then ALERT again: the second firing episode is a + // fresh transition, so it notifies even with re-notification off. + expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(3); + }); + }); }); describe('processAlert with materialized views', () => { diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index a5259b39de..654bf30477 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -773,6 +773,18 @@ export const parseAlertData = ( return { value, extraFields }; }; +/** + * Whether the group already has a notification that was sent and has not been + * resolved yet. `fired` is absent on histories written before it was + * introduced, so only an explicit `false` counts as "never notified". + */ +const hasUnresolvedNotification = ( + previousHistory: AggregatedAlertHistory | undefined, +): boolean => + (previousHistory?.state === AlertState.ALERT || + previousHistory?.state === AlertState.PENDING) && + previousHistory.fired !== false; + export const processAlert = async ( now: Date, details: AlertDetails, @@ -1057,7 +1069,7 @@ export const processAlert = async ( }, 'Skipped firing alert due to silence', ); - return; + return false; } alertEvaluationsCounter.add(1, { @@ -1100,6 +1112,37 @@ export const processAlert = async ( ); executionErrors.push(makeWebhookAlertError(e)); } + return true; + }; + + // Gates re-notification of an already-firing group by renotifyIntervalMinutes. + const shouldNotifyWhileFiring = ( + previousHistory: AggregatedAlertHistory | undefined, + ): boolean => { + if (!hasUnresolvedNotification(previousHistory)) { + return true; + } + const { renotifyIntervalMinutes } = alert; + const lastNotifiedAt = previousHistory?.lastNotifiedAt; + const shouldNotify = + renotifyIntervalMinutes != null && + (renotifyIntervalMinutes <= 0 || + lastNotifiedAt == null || + nowInMinsRoundDown.getTime() - lastNotifiedAt.getTime() >= + renotifyIntervalMinutes * 60_000); + + if (!shouldNotify) { + alertEvaluationsCounter.add(1, { outcome: 'skipped_renotify' }); + logger.debug( + { + alertId: alert.id, + lastNotifiedAt, + renotifyIntervalMinutes, + }, + 'Skipped re-notifying an already-notified alert', + ); + } + return shouldNotify; }; const numWindowsToLookBack = alert.numConsecutiveWindows ?? 1; @@ -1131,9 +1174,7 @@ export const processAlert = async ( groupKey: string, ) => { if ( - (previousHistory?.state === AlertState.ALERT || - previousHistory?.state === AlertState.PENDING) && - previousHistory?.fired !== false && + hasUnresolvedNotification(previousHistory) && currentHistory.state === AlertState.OK ) { const lastValue = @@ -1170,17 +1211,23 @@ export const processAlert = async ( history.lastValues.push({ count: value, startTime: alertTimestamp }); const previous = previousMap.get(computeHistoryMapKey(alert.id, '')); + history.lastNotifiedAt = previous?.lastNotifiedAt; if (doesExceedThreshold(alert, value)) { history.counts += 1; if (shouldFireBasedOnConsecutiveWindows()) { history.state = AlertState.ALERT; history.fired = true; - await trySendNotification({ - state: AlertState.ALERT, - group: '', - totalCount: value, - startTime: alertTimestamp, - }); + if ( + shouldNotifyWhileFiring(previous) && + (await trySendNotification({ + state: AlertState.ALERT, + group: '', + totalCount: value, + startTime: alertTimestamp, + })) + ) { + history.lastNotifiedAt = nowInMinsRoundDown; + } } else { history.state = AlertState.PENDING; // Carry forward fired=true if a notification was previously sent and not yet resolved. @@ -1376,18 +1423,23 @@ export const processAlert = async ( const hitAlertThisRun = latestAlertContext.has(groupKey); const wasAlertingBefore = groupPrevious?.state === AlertState.ALERT; + history.lastNotifiedAt = groupPrevious?.lastNotifiedAt; - // If it hit ALERT during this run, send the notification (re-notifying every tick if it continuously breaches) + // If it hit ALERT during this run, send the notification if (hitAlertThisRun) { const context = latestAlertContext.get(groupKey); - if (context) { - await trySendNotification({ - state: AlertState.ALERT, - group: groupKey, - totalCount: context.value, - startTime: context.startTime, - attributes: context.attributes, - }); + if (context && shouldNotifyWhileFiring(groupPrevious)) { + if ( + await trySendNotification({ + state: AlertState.ALERT, + group: groupKey, + totalCount: context.value, + startTime: context.startTime, + attributes: context.attributes, + }) + ) { + history.lastNotifiedAt = nowInMinsRoundDown; + } // Inject a mock previous history so the resolve check below catches it // if the final state for this group is OK (i.e. it breached then resolved). @@ -1464,6 +1516,7 @@ export interface AggregatedAlertHistory { state: AlertState; group?: string; fired?: boolean; + lastNotifiedAt?: Date; } /** @@ -1521,6 +1574,7 @@ export const getPreviousAlertHistories = async ( createdAt: { $first: '$createdAt' }, state: { $first: '$state' }, fired: { $first: '$fired' }, + lastNotifiedAt: { $first: '$lastNotifiedAt' }, }, }, { @@ -1530,6 +1584,7 @@ export const getPreviousAlertHistories = async ( state: 1, group: '$_id.group', fired: 1, + lastNotifiedAt: 1, }, }, ]); diff --git a/packages/api/src/utils/externalApi.ts b/packages/api/src/utils/externalApi.ts index c25e76cbcc..5ef9f269ec 100644 --- a/packages/api/src/utils/externalApi.ts +++ b/packages/api/src/utils/externalApi.ts @@ -238,6 +238,7 @@ export type ExternalAlert = { scheduleOffsetMinutes?: number; scheduleStartAt?: string | null; numConsecutiveWindows?: number | null; + renotifyIntervalMinutes?: number | null; thresholdType: AlertThresholdType; source?: string; state: AlertState; @@ -340,6 +341,7 @@ export function translateAlertDocumentToExternalAlert( }), scheduleStartAt: transformScheduleStartAt(alertObj.scheduleStartAt), numConsecutiveWindows: alertObj.numConsecutiveWindows ?? null, + renotifyIntervalMinutes: alertObj.renotifyIntervalMinutes ?? null, thresholdType: alertObj.thresholdType, source: alertObj.source, state: alertObj.state, diff --git a/packages/api/src/utils/zod.ts b/packages/api/src/utils/zod.ts index 42f11afeab..f8a77be31e 100644 --- a/packages/api/src/utils/zod.ts +++ b/packages/api/src/utils/zod.ts @@ -672,6 +672,7 @@ export const alertSchema = z message: z.string().min(1).max(4096).nullish(), note: alertNoteSchema, numConsecutiveWindows: z.number().int().min(1).nullish(), + renotifyIntervalMinutes: z.number().int().min(0).nullish(), }) .and(zSavedSearchAlert.or(zTileAlert)) .superRefine(validateAlertScheduleOffsetMinutes) diff --git a/packages/app/src/DBSearchPageAlertModal.tsx b/packages/app/src/DBSearchPageAlertModal.tsx index 34c994fce5..4562dd417d 100644 --- a/packages/app/src/DBSearchPageAlertModal.tsx +++ b/packages/app/src/DBSearchPageAlertModal.tsx @@ -80,6 +80,7 @@ const SavedSearchAlertFormSchema = z // nullish() (not optional()): persisted alerts store this as null, which // optional() would reject. numConsecutiveWindows: z.number().int().min(1).nullish(), + renotifyIntervalMinutes: z.number().int().min(0).nullish(), }) .passthrough() .superRefine(validateAlertScheduleOffsetMinutes) @@ -128,6 +129,8 @@ const AlertForm = ({ // Persisted null -> undefined for the NumberInput. numConsecutiveWindows: defaultValues.numConsecutiveWindows ?? undefined, + renotifyIntervalMinutes: + defaultValues.renotifyIntervalMinutes ?? undefined, } : { interval: '5m', @@ -160,6 +163,10 @@ const AlertForm = ({ control, name: 'numConsecutiveWindows', }); + const renotifyIntervalMinutes = useWatch({ + control, + name: 'renotifyIntervalMinutes', + }); const maxScheduleOffsetMinutes = Math.max( intervalToMinutes(interval ?? '5m') - 1, 0, @@ -284,6 +291,8 @@ const AlertForm = ({ offsetWindowLabel={`from each ${intervalLabel} window`} numConsecutiveWindowsName="numConsecutiveWindows" numConsecutiveWindows={numConsecutiveWindows ?? undefined} + renotifyIntervalName="renotifyIntervalMinutes" + renotifyIntervalMinutes={renotifyIntervalMinutes ?? undefined} /> grouped by diff --git a/packages/app/src/components/AlertScheduleFields.tsx b/packages/app/src/components/AlertScheduleFields.tsx index 774baca58a..b089a6fe7d 100644 --- a/packages/app/src/components/AlertScheduleFields.tsx +++ b/packages/app/src/components/AlertScheduleFields.tsx @@ -38,6 +38,8 @@ type AlertScheduleFieldsProps = { offsetWindowLabel: string; numConsecutiveWindowsName?: FieldPath; numConsecutiveWindows?: number; + renotifyIntervalName?: FieldPath; + renotifyIntervalMinutes?: number; }; export function AlertScheduleFields({ @@ -50,6 +52,8 @@ export function AlertScheduleFields({ offsetWindowLabel, numConsecutiveWindowsName, numConsecutiveWindows, + renotifyIntervalName, + renotifyIntervalMinutes, }: AlertScheduleFieldsProps) { const showScheduleOffsetInput = maxScheduleOffsetMinutes > 0; const scheduleStartAtValue = useWatch({ @@ -60,7 +64,8 @@ export function AlertScheduleFields({ const hasAdvancedScheduleValues = (scheduleOffsetMinutes ?? 0) > 0 || hasScheduleStartAtAnchor || - (numConsecutiveWindows ?? 1) > 1; + (numConsecutiveWindows ?? 1) > 1 || + renotifyIntervalMinutes != null; const [opened, setOpened] = useState(hasAdvancedScheduleValues); useEffect(() => { @@ -149,6 +154,46 @@ export function AlertScheduleFields({ )} + {renotifyIntervalName && ( + + + + Re-notify every + + + + + + + + ( + + field.onChange(typeof v === 'number' ? v : undefined) + } + min={0} + placeholder="Never" + size="xs" + w={90} + /> + )} + /> + + minutes + + + )} {showScheduleOffsetInput && ( <> diff --git a/packages/app/src/components/DBEditTimeChartForm/TileAlertEditor.tsx b/packages/app/src/components/DBEditTimeChartForm/TileAlertEditor.tsx index 8bfb0180fa..9104bf99e1 100644 --- a/packages/app/src/components/DBEditTimeChartForm/TileAlertEditor.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/TileAlertEditor.tsx @@ -76,6 +76,10 @@ export function TileAlertEditor({ control, name: 'alert.numConsecutiveWindows', }); + const alertRenotifyIntervalMinutes = useWatch({ + control, + name: 'alert.renotifyIntervalMinutes', + }); const maxAlertScheduleOffsetMinutes = alert?.interval ? Math.max(intervalToMinutes(alert.interval) - 1, 0) : 0; @@ -243,6 +247,8 @@ export function TileAlertEditor({ } numConsecutiveWindowsName="alert.numConsecutiveWindows" numConsecutiveWindows={alertnumConsecutiveWindows ?? undefined} + renotifyIntervalName="alert.renotifyIntervalMinutes" + renotifyIntervalMinutes={alertRenotifyIntervalMinutes ?? undefined} /> Send to diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 9ef47c1da4..814544d83e 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -774,6 +774,7 @@ export const AlertBaseObjectSchema = z.object({ }) .optional(), numConsecutiveWindows: z.number().int().min(1).nullish(), + renotifyIntervalMinutes: z.number().int().min(0).nullish(), }); // Keep AlertBaseSchema as a ZodObject for backwards compatibility with @@ -2144,6 +2145,7 @@ export const AlertsPageItemSchema = z.object({ .optional(), executionErrors: z.array(AlertErrorSchema).optional(), numConsecutiveWindows: z.number().int().min(1).nullish(), + renotifyIntervalMinutes: z.number().int().min(0).nullish(), }); export type AlertsPageItem = z.infer; From 07d077a0bd8a7e75cee1f544d0f03fa9c3482feb Mon Sep 17 00:00:00 2001 From: Niladri Adhikary Date: Thu, 30 Jul 2026 12:22:22 +0530 Subject: [PATCH 2/4] chore: refactoring comments Signed-off-by: Niladri Adhikary --- .../__tests__/checkAlerts.int.test.ts | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index b7c17873d3..18b7c3bebc 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -9098,9 +9098,7 @@ describe('checkAlerts', () => { }); describe('re-notification (renotifyIntervalMinutes)', () => { - // A sustained breach across four consecutive 5m windows: every evaluation - // below sees data in its own window, so the alert stays in ALERT - // throughout without ever returning to OK. + // Data in every 5m window, so the alert stays ALERT and never resolves. const insertSustainedBreach = () => bulkInsertLogs( ['00:05:00', '00:10:00', '00:15:00', '00:20:00'].map(time => ({ @@ -9143,8 +9141,7 @@ describe('checkAlerts', () => { channel: { type: 'webhook', webhookId: webhook._id.toString() }, interval: '5m', thresholdType: AlertThresholdType.ABOVE, - // threshold=1 (not 0): ABOVE is inclusive, so threshold=0 would make - // an empty window breach too and the alert could never resolve. + // threshold=1: ABOVE is inclusive, so 0 would breach empty windows too. threshold: 1, savedSearchId: savedSearch.id, renotifyIntervalMinutes, @@ -9188,8 +9185,7 @@ describe('checkAlerts', () => { 'ALERT', 'ALERT', ]); - // lastNotifiedAt is pinned to the window that notified and carried - // forward unchanged across the silent evaluations. + // lastNotifiedAt stays pinned to the notifying window while silent. expect(histories.map(h => h.lastNotifiedAt?.toISOString())).toEqual( Array(4).fill('2024-01-01T00:10:00.000Z'), ); @@ -9208,8 +9204,7 @@ describe('checkAlerts', () => { }); it('re-notifies only after renotifyIntervalMinutes has elapsed', async () => { - // 10m re-notification on a 5m interval: notify at 00:10, skip 00:15, - // re-notify at 00:20, skip 00:25. + // 10m interval, 5m ticks: notify 00:10, skip 00:15, notify 00:20, skip 00:25. const { notificationCounts, histories } = await runSustainedBreach(10); expect(notificationCounts).toEqual([1, 1, 2, 2]); @@ -9276,8 +9271,7 @@ describe('checkAlerts', () => { }).sort({ createdAt: 1 }); expect(histories.map(h => h.state)).toEqual(['ALERT', 'OK', 'ALERT']); - // ALERT, resolved OK, then ALERT again: the second firing episode is a - // fresh transition, so it notifies even with re-notification off. + // Second ALERT is a fresh transition after resolving, so it notifies. expect(slack.postMessageToWebhook).toHaveBeenCalledTimes(3); }); }); From 94e4f747bd7bde1ff827a5acc9f88b1bfbd560d9 Mon Sep 17 00:00:00 2001 From: Niladri Adhikary Date: Thu, 30 Jul 2026 12:33:34 +0530 Subject: [PATCH 3/4] fix: corrected return Signed-off-by: Niladri Adhikary --- packages/api/src/tasks/checkAlerts/index.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 654bf30477..288a0aebff 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -1111,6 +1111,7 @@ export const processAlert = async ( 'Failed to fire channel event', ); executionErrors.push(makeWebhookAlertError(e)); + return false; } return true; }; @@ -1125,11 +1126,11 @@ export const processAlert = async ( const { renotifyIntervalMinutes } = alert; const lastNotifiedAt = previousHistory?.lastNotifiedAt; const shouldNotify = - renotifyIntervalMinutes != null && - (renotifyIntervalMinutes <= 0 || - lastNotifiedAt == null || - nowInMinsRoundDown.getTime() - lastNotifiedAt.getTime() >= - renotifyIntervalMinutes * 60_000); + lastNotifiedAt == null || + (renotifyIntervalMinutes != null && + (renotifyIntervalMinutes <= 0 || + nowInMinsRoundDown.getTime() - lastNotifiedAt.getTime() >= + renotifyIntervalMinutes * 60_000)); if (!shouldNotify) { alertEvaluationsCounter.add(1, { outcome: 'skipped_renotify' }); From c316ffad739dfa9dcad2e8632951043b9f774084 Mon Sep 17 00:00:00 2001 From: Niladri Adhikary Date: Sat, 1 Aug 2026 07:42:36 +0530 Subject: [PATCH 4/4] fix: missing field in test Signed-off-by: Niladri Adhikary --- .../api/src/routers/external-api/__tests__/alerts.int.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/src/routers/external-api/__tests__/alerts.int.test.ts b/packages/api/src/routers/external-api/__tests__/alerts.int.test.ts index f241e405ef..8817e524ef 100644 --- a/packages/api/src/routers/external-api/__tests__/alerts.int.test.ts +++ b/packages/api/src/routers/external-api/__tests__/alerts.int.test.ts @@ -224,6 +224,7 @@ describe('External API Alerts', () => { message: 'This is a test alert for format verification', note: null, numConsecutiveWindows: null, + renotifyIntervalMinutes: null, threshold: 123, interval: '15m', source: AlertSource.TILE,