From e0784dec47148be2b4bfb6e6cd16cf94ad276f32 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Thu, 6 Aug 2026 14:33:04 -0700 Subject: [PATCH 1/3] =?UTF-8?q?feat(alerts):=20evaluations=20read=20model?= =?UTF-8?q?=20=E2=80=94=20ERROR=20rows,=20analytics,=20per-group=20windows?= =?UTF-8?q?=20(HDX-4997)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AlertHistory read-side support for the alert detail page: - Types (common-utils) for evaluation errors (AlertError/AlertErrorType incl. QUERY_TIMEOUT), per-window evaluations with per-group breakdown (capped at ALERT_EVALUATION_GROUPS_LIMIT, firing-first), and evaluation analytics (queryDurationMs, webhookDurationMs, backfilledBuckets). - AlertHistory schema gains optional errors + analytics fields, and AlertState gains ERROR (only ever used on history rows). - GET /alerts/:id/evaluations: per-window evaluation history scoped to a startTime/endTime range (clamped to the 31d retention window), grouped across group-by groups newest-first with a hard-bounded scan of at most ~(limit+1) intervals per request and a server-provided nextBefore cursor that always advances past the scanned slice, so paging progresses across gaps instead of stalling. - Windows with ERROR rows surface their errors (deduped, newest-first) and rank as ERROR; firing-transition annotations exclude ERROR rows. Nothing writes ERROR rows or analytics yet — the alert task's write side lands separately. --- .changeset/alert-detail-page-evaluations.md | 13 + .../__tests__/alertHistory.int.test.ts | 540 ++++++++++++++++++ packages/api/src/controllers/alertHistory.ts | 383 ++++++++++++- packages/api/src/models/alert.ts | 7 + packages/api/src/models/alertHistory.ts | 61 +- .../routers/api/__tests__/alerts.int.test.ts | 226 ++++++++ packages/api/src/routers/api/alerts.ts | 91 ++- packages/common-utils/src/types.ts | 79 +++ 8 files changed, 1371 insertions(+), 29 deletions(-) create mode 100644 .changeset/alert-detail-page-evaluations.md diff --git a/.changeset/alert-detail-page-evaluations.md b/.changeset/alert-detail-page-evaluations.md new file mode 100644 index 0000000000..c5d83c78e5 --- /dev/null +++ b/.changeset/alert-detail-page-evaluations.md @@ -0,0 +1,13 @@ +--- +'@hyperdx/common-utils': minor +'@hyperdx/api': minor +'@hyperdx/app': minor +--- + +Add an alert detail page (/alerts/:id) with the alert's query charted against +its threshold, a widened evaluation-history strip, and a paginated evaluation +event stream (per-group breakdown for group-by alerts, evaluation analytics +columns, time-range-scoped cursor pagination). Adds the +GET /alerts/:id/evaluations endpoint and the AlertHistory read-side support +for ERROR-state rows and evaluation analytics; the alerts page history strip +renders errored evaluation windows with per-window error details. diff --git a/packages/api/src/controllers/__tests__/alertHistory.int.test.ts b/packages/api/src/controllers/__tests__/alertHistory.int.test.ts index 1539410493..71a1a31ade 100644 --- a/packages/api/src/controllers/__tests__/alertHistory.int.test.ts +++ b/packages/api/src/controllers/__tests__/alertHistory.int.test.ts @@ -1,6 +1,8 @@ import { ObjectId } from 'mongodb'; import { + ALERT_EVALUATION_GROUPS_LIMIT, + getAlertEvaluations, getAlertTransitionsInRange, getRecentAlertHistories, getRecentAlertHistoriesBatch, @@ -368,6 +370,520 @@ describe('alertHistory controller', () => { expect(histories[0].state).toBe(AlertState.ALERT); expect(histories[0].counts).toBe(5); }); + + it('surfaces ERROR windows with their recorded errors', async () => { + const team = await Team.create({ name: 'Test Team' }); + const alert = await Alert.create({ + team: team._id, + threshold: 100, + interval: '5m', + channel: { type: null }, + }); + + const errorWindow = new Date(Date.now() - 60000); + const okWindow = new Date(Date.now() - 120000); + const errorTimestamp = new Date(Date.now() - 55000); + + await AlertHistory.create({ + alert: alert._id, + createdAt: errorWindow, + state: AlertState.ERROR, + counts: 0, + lastValues: [], + errors: [ + { + timestamp: errorTimestamp, + type: 'QUERY_TIMEOUT', + message: 'Alert query did not complete within the 300s timeout', + }, + ], + }); + await AlertHistory.create({ + alert: alert._id, + createdAt: okWindow, + state: AlertState.OK, + counts: 0, + lastValues: [{ startTime: okWindow, count: 0 }], + }); + + const histories = await getRecentAlertHistories({ + alertId: new ObjectId(alert._id), + interval: '5m', + limit: 10, + }); + + expect(histories).toHaveLength(2); + expect(histories[0].state).toBe(AlertState.ERROR); + expect(histories[0].errors).toHaveLength(1); + expect(histories[0].errors![0].type).toBe('QUERY_TIMEOUT'); + expect(histories[0].errors![0].message).toContain('300s timeout'); + expect(histories[1].state).toBe(AlertState.OK); + expect(histories[1].errors).toBeUndefined(); + }); + + it('lets ALERT/PENDING outrank ERROR within a grouped window, but ERROR outrank OK', async () => { + const team = await Team.create({ name: 'Test Team' }); + const alert = await Alert.create({ + team: team._id, + threshold: 100, + interval: '5m', + channel: { type: null }, + }); + + const alertWindow = new Date(Date.now() - 60000); + const okWindow = new Date(Date.now() - 120000); + const makeError = () => ({ + timestamp: new Date(), + type: 'WEBHOOK_ERROR', + message: 'Failed to send webhook notification.', + }); + + // Window that fired AND recorded a notification error → shows ALERT + await AlertHistory.create({ + alert: alert._id, + createdAt: alertWindow, + state: AlertState.ALERT, + counts: 2, + lastValues: [{ startTime: alertWindow, count: 2 }], + }); + await AlertHistory.create({ + alert: alert._id, + createdAt: alertWindow, + state: AlertState.ERROR, + counts: 0, + lastValues: [], + errors: [makeError()], + }); + + // Window that was OK but the resolve notification failed → shows ERROR + await AlertHistory.create({ + alert: alert._id, + createdAt: okWindow, + state: AlertState.OK, + counts: 0, + lastValues: [{ startTime: okWindow, count: 0 }], + }); + await AlertHistory.create({ + alert: alert._id, + createdAt: okWindow, + state: AlertState.ERROR, + counts: 0, + lastValues: [], + errors: [makeError()], + }); + + const histories = await getRecentAlertHistories({ + alertId: new ObjectId(alert._id), + interval: '5m', + limit: 10, + }); + + expect(histories).toHaveLength(2); + expect(histories[0].state).toBe(AlertState.ALERT); + // Errors from the ERROR row are still surfaced on the merged window + expect(histories[0].errors).toHaveLength(1); + expect(histories[1].state).toBe(AlertState.ERROR); + expect(histories[1].errors).toHaveLength(1); + }); + }); + + describe('getAlertEvaluations', () => { + const createAlert = async () => { + const team = await Team.create({ name: 'Test Team' }); + return Alert.create({ + team: team._id, + threshold: 100, + interval: '5m', + channel: { type: null }, + }); + }; + + const createOkWindow = (alertId: any, createdAt: Date) => + AlertHistory.create({ + alert: alertId, + createdAt, + state: AlertState.OK, + counts: 0, + lastValues: [{ startTime: createdAt, count: 0 }], + }); + + it('only returns windows within [startTime, endTime]', async () => { + const alert = await createAlert(); + const now = Date.now(); + const at = (minsAgo: number) => new Date(now - minsAgo * 60_000); + + await createOkWindow(alert._id, at(5)); // after endTime + await createOkWindow(alert._id, at(15)); // in range + await createOkWindow(alert._id, at(20)); // in range + await createOkWindow(alert._id, at(40)); // before startTime + + const page = await getAlertEvaluations({ + alertId: new ObjectId(alert._id), + interval: '5m', + limit: 10, + startTime: at(30), + endTime: at(10), + }); + + expect(page.data).toHaveLength(2); + expect(page.data[0].createdAt).toEqual(at(15)); + expect(page.data[1].createdAt).toEqual(at(20)); + expect(page.hasMore).toBe(false); + expect(page.nextBefore).toBeUndefined(); + }); + + it('paginates older windows via the nextBefore cursor when the page fills up', async () => { + const alert = await createAlert(); + const now = Date.now(); + const windows = [1, 2, 3, 4].map(i => new Date(now - i * 5 * 60_000)); + for (const createdAt of windows) { + await createOkWindow(alert._id, createdAt); + } + + const startTime = new Date(now - 60 * 60_000); + const endTime = new Date(now); + + const firstPage = await getAlertEvaluations({ + alertId: new ObjectId(alert._id), + interval: '5m', + limit: 2, + startTime, + endTime, + }); + expect(firstPage.data).toHaveLength(2); + expect(firstPage.data[0].createdAt).toEqual(windows[0]); + expect(firstPage.data[1].createdAt).toEqual(windows[1]); + expect(firstPage.hasMore).toBe(true); + expect(firstPage.nextBefore).toEqual(windows[1]); + + const secondPage = await getAlertEvaluations({ + alertId: new ObjectId(alert._id), + interval: '5m', + limit: 2, + startTime, + endTime, + before: firstPage.nextBefore, + }); + expect(secondPage.data).toHaveLength(2); + expect(secondPage.data[0].createdAt).toEqual(windows[2]); + expect(secondPage.data[1].createdAt).toEqual(windows[3]); + }); + + it('keeps paging across gaps: the scan bound truncates but nextBefore advances', async () => { + const alert = await createAlert(); + const now = Date.now(); + const at = (minsAgo: number) => new Date(now - minsAgo * 60_000); + + // One recent window, then a gap far wider than the per-request scan + // bound ((limit + 1) × interval = 3 minutes for limit=2 / 1m interval), + // then an old window still inside the requested range. + await createOkWindow(alert._id, at(1)); + await createOkWindow(alert._id, at(20)); + + const startTime = at(30); + const endTime = at(0); + + // First page: finds the recent window; the scan bound stops long + // before startTime, so more may exist. + const firstPage = await getAlertEvaluations({ + alertId: new ObjectId(alert._id), + interval: '1m', + limit: 2, + startTime, + endTime, + }); + expect(firstPage.data).toHaveLength(1); + expect(firstPage.data[0].createdAt).toEqual(at(1)); + expect(firstPage.hasMore).toBe(true); + expect(firstPage.nextBefore).toBeDefined(); + + // Follow the cursor until the old window is found or the range is + // exhausted. Every hop advances by at least one scan bound, so this + // terminates. + let before = firstPage.nextBefore; + let found: Date | undefined; + for (let i = 0; i < 20 && before != null; i++) { + const page = await getAlertEvaluations({ + alertId: new ObjectId(alert._id), + interval: '1m', + limit: 2, + startTime, + endTime, + before, + }); + if (page.data.length > 0) { + found = page.data[0].createdAt; + break; + } + expect(page.hasMore).toBe(true); + // Empty pages still advance the cursor + expect(page.nextBefore!.getTime()).toBeLessThan(before.getTime()); + before = page.nextBefore; + } + expect(found).toEqual(at(20)); + }); + + it('reports hasMore=false once the scan reaches startTime', async () => { + const alert = await createAlert(); + const now = Date.now(); + const at = (minsAgo: number) => new Date(now - minsAgo * 60_000); + + await createOkWindow(alert._id, at(5)); + + const page = await getAlertEvaluations({ + alertId: new ObjectId(alert._id), + interval: '5m', + limit: 10, + startTime: at(20), + endTime: at(0), + }); + + expect(page.data).toHaveLength(1); + expect(page.hasMore).toBe(false); + expect(page.nextBefore).toBeUndefined(); + // Non-grouped rows carry no per-group breakdown + expect(page.data[0].groups).toBeUndefined(); + expect(page.data[0].groupsTotal).toBeUndefined(); + }); + + it('breaks grouped windows down per group, firing-first', async () => { + const alert = await createAlert(); + const now = Date.now(); + const windowStart = new Date(now - 5 * 60_000); + const bucket = new Date(now - 10 * 60_000); + + const createGroupRow = ( + group: string, + state: AlertState, + count: number, + fired?: boolean, + ) => + AlertHistory.create({ + alert: alert._id, + createdAt: windowStart, + state, + counts: state === AlertState.OK ? 0 : 1, + lastValues: [{ startTime: bucket, count }], + group, + ...(fired != null && { fired }), + }); + + await createGroupRow('ServiceName:web', AlertState.OK, 3); + await createGroupRow('ServiceName:api', AlertState.ALERT, 14, true); + await createGroupRow('ServiceName:worker', AlertState.PENDING, 9); + // A notification failure recorded for the window: contributes errors, + // never a group entry. + await AlertHistory.create({ + alert: alert._id, + createdAt: windowStart, + state: AlertState.ERROR, + counts: 0, + lastValues: [], + errors: [ + { + timestamp: new Date(), + type: 'WEBHOOK_ERROR', + message: 'Failed to send webhook notification.', + }, + ], + }); + + const page = await getAlertEvaluations({ + alertId: new ObjectId(alert._id), + interval: '5m', + limit: 10, + startTime: new Date(now - 60 * 60_000), + endTime: new Date(now), + }); + + expect(page.data).toHaveLength(1); + const window = page.data[0]; + // Overall window state merges the per-group states + expect(window.state).toBe(AlertState.ALERT); + expect(window.groupsTotal).toBe(3); + expect(window.groups).toHaveLength(3); + // Firing-first ordering: ALERT > PENDING > OK + expect(window.groups!.map(g => g.group)).toEqual([ + 'ServiceName:api', + 'ServiceName:worker', + 'ServiceName:web', + ]); + expect(window.groups![0]).toMatchObject({ + group: 'ServiceName:api', + state: AlertState.ALERT, + counts: 1, + fired: true, + }); + expect(window.groups![0].lastValue).toMatchObject({ count: 14 }); + expect(window.groups![2]).toMatchObject({ + group: 'ServiceName:web', + state: AlertState.OK, + counts: 0, + }); + // The ERROR row surfaces as window errors, not as a group + expect(window.errors).toHaveLength(1); + expect(window.groups!.every(g => g.state !== AlertState.ERROR)).toBe( + true, + ); + }); + + it('surfaces evaluation analytics, preferring the successful evaluation over a failed attempt', async () => { + const alert = await createAlert(); + const now = Date.now(); + const windowStart = new Date(now - 5 * 60_000); + const bucket = new Date(now - 10 * 60_000); + + // Failed first attempt for this window (its query ran 300s) + await AlertHistory.create({ + alert: alert._id, + createdAt: windowStart, + state: AlertState.ERROR, + counts: 0, + lastValues: [], + errors: [ + { + timestamp: new Date(), + type: 'QUERY_TIMEOUT', + message: 'timed out', + }, + ], + analytics: { queryDurationMs: 300_000 }, + }); + // Successful retry + await AlertHistory.create({ + alert: alert._id, + createdAt: windowStart, + state: AlertState.OK, + counts: 0, + lastValues: [{ startTime: bucket, count: 1 }], + analytics: { + queryDurationMs: 1_200, + webhookDurationMs: 340, + backfilledBuckets: 0, + }, + }); + + const page = await getAlertEvaluations({ + alertId: new ObjectId(alert._id), + interval: '5m', + limit: 10, + startTime: new Date(now - 60 * 60_000), + endTime: new Date(now), + }); + + expect(page.data).toHaveLength(1); + expect(page.data[0].analytics).toMatchObject({ + queryDurationMs: 1_200, + webhookDurationMs: 340, + backfilledBuckets: 0, + }); + }); + + it('falls back to the failed attempt analytics when no successful row exists', async () => { + const alert = await createAlert(); + const now = Date.now(); + const windowStart = new Date(now - 5 * 60_000); + + await AlertHistory.create({ + alert: alert._id, + createdAt: windowStart, + state: AlertState.ERROR, + counts: 0, + lastValues: [], + errors: [ + { + timestamp: new Date(), + type: 'QUERY_TIMEOUT', + message: 'timed out', + }, + ], + analytics: { queryDurationMs: 300_000 }, + }); + + const page = await getAlertEvaluations({ + alertId: new ObjectId(alert._id), + interval: '5m', + limit: 10, + startTime: new Date(now - 60 * 60_000), + endTime: new Date(now), + }); + + expect(page.data).toHaveLength(1); + expect(page.data[0].analytics).toMatchObject({ + queryDurationMs: 300_000, + // Derived from lastValues (none) for rows lacking the field + backfilledBuckets: 0, + }); + }); + + it('derives backfilled buckets from lastValues for rows written before analytics existed', async () => { + const alert = await createAlert(); + const now = Date.now(); + const windowStart = new Date(now - 5 * 60_000); + const bucketAt = (minsAgo: number) => new Date(now - minsAgo * 60_000); + + // Legacy backfill row: one evaluation covering three buckets, no + // analytics field. + await AlertHistory.create({ + alert: alert._id, + createdAt: windowStart, + state: AlertState.OK, + counts: 0, + lastValues: [ + { startTime: bucketAt(20), count: 0 }, + { startTime: bucketAt(15), count: 0 }, + { startTime: bucketAt(10), count: 0 }, + ], + }); + + const page = await getAlertEvaluations({ + alertId: new ObjectId(alert._id), + interval: '5m', + limit: 10, + startTime: new Date(now - 60 * 60_000), + endTime: new Date(now), + }); + + expect(page.data).toHaveLength(1); + expect(page.data[0].analytics).toEqual({ backfilledBuckets: 2 }); + }); + + it('caps the per-group breakdown and reports the pre-cap total', async () => { + const alert = await createAlert(); + const now = Date.now(); + const windowStart = new Date(now - 5 * 60_000); + const bucket = new Date(now - 10 * 60_000); + + const total = ALERT_EVALUATION_GROUPS_LIMIT + 5; + // One firing group buried among many OK groups: firing-first ordering + // must keep it visible after the cap. + const rows = Array.from({ length: total }, (_, i) => ({ + alert: alert._id, + createdAt: windowStart, + state: i === total - 1 ? AlertState.ALERT : AlertState.OK, + counts: i === total - 1 ? 1 : 0, + lastValues: [{ startTime: bucket, count: i }], + group: `ServiceName:svc-${String(i).padStart(3, '0')}`, + })); + await AlertHistory.insertMany(rows); + + const page = await getAlertEvaluations({ + alertId: new ObjectId(alert._id), + interval: '5m', + limit: 10, + startTime: new Date(now - 60 * 60_000), + endTime: new Date(now), + }); + + expect(page.data).toHaveLength(1); + const window = page.data[0]; + expect(window.groupsTotal).toBe(total); + expect(window.groups).toHaveLength(ALERT_EVALUATION_GROUPS_LIMIT); + expect(window.groups![0].state).toBe(AlertState.ALERT); + expect(window.groups![0].group).toBe( + `ServiceName:svc-${String(total - 1).padStart(3, '0')}`, + ); + }); }); describe('getRecentAlertHistoriesBatch', () => { @@ -703,6 +1219,30 @@ describe('alertHistory controller', () => { expect(transitions[1].createdAt).toBe(t(15).toISOString()); }); + it('ignores ERROR windows so a failed evaluation mid-firing is not a recovery', async () => { + const alert = await createAlert(); + await createHistory(alert._id, t(30), AlertState.OK, 0); + await createHistory(alert._id, t(25), AlertState.ALERT, 3); + // Evaluation failed mid-episode — must not read as a recovery + refire + await createHistory(alert._id, t(20), AlertState.ERROR, 0); + await createHistory(alert._id, t(15), AlertState.ALERT, 4); + await createHistory(alert._id, t(10), AlertState.OK, 0); + + const transitions = await getAlertTransitionsInRange({ + alertId: new ObjectId(alert._id), + interval: '5m', + startTime: t(40), + endTime: t(0), + }); + + expect(transitions.map(tr => tr.state)).toEqual([ + AlertState.ALERT, + AlertState.OK, + ]); + expect(transitions[0].createdAt).toBe(t(25).toISOString()); + expect(transitions[1].createdAt).toBe(t(10).toISOString()); + }); + it('only considers history for the specified alert', async () => { const alert1 = await createAlert(); const alert2 = await createAlert(); diff --git a/packages/api/src/controllers/alertHistory.ts b/packages/api/src/controllers/alertHistory.ts index 68caff4749..a3552c3274 100644 --- a/packages/api/src/controllers/alertHistory.ts +++ b/packages/api/src/controllers/alertHistory.ts @@ -1,22 +1,36 @@ import PQueue from '@esm2cjs/p-queue'; import { + ALERT_EVALUATION_GROUPS_LIMIT, ALERT_INTERVAL_TO_MINUTES, AlertInterval, AlertTransition, } from '@hyperdx/common-utils/dist/types'; import { ObjectId } from 'mongodb'; -import { AlertState } from '@/models/alert'; -import AlertHistory, { IAlertHistory } from '@/models/alertHistory'; +import { AlertState, IAlertError } from '@/models/alert'; +import AlertHistory, { + IAlertHistory, + IAlertHistoryAnalytics, +} from '@/models/alertHistory'; + +// Re-exported for API-side consumers/tests; the app imports it from +// common-utils to explain the cap in the UI. +export { ALERT_EVALUATION_GROUPS_LIMIT }; // Max parallel per-alert queries to avoid overwhelming the DB connection pool export const ALERT_HISTORY_QUERY_CONCURRENCY = 20; +/** Alert evaluation interval in milliseconds. */ +const intervalToMs = (interval: AlertInterval): number => + // eslint-disable-next-line security/detect-object-injection -- interval is a typed AlertInterval union, not user input + ALERT_INTERVAL_TO_MINUTES[interval] * 60 * 1000; + type GroupedAlertHistory = { _id: Date; states: string[]; counts: number; lastValues: IAlertHistory['lastValues'][]; + errors: IAlertError[][]; }; function groupStateToOverallState(states: string[]): AlertState { @@ -28,25 +42,103 @@ function groupStateToOverallState(states: string[]): AlertState { return AlertState.PENDING; } + // An evaluation window that neither fired nor was pending, but recorded an + // error (query failure, or a notification failure on an OK window), is + // surfaced as ERROR. + if (states.includes(AlertState.ERROR)) { + return AlertState.ERROR; + } + return AlertState.OK; } +/** Dedupe errors by type+message, keeping the most recent occurrence. */ +function dedupeErrors(errors: IAlertError[]): IAlertError[] { + const map = new Map(); + for (const error of errors) { + const key = `${error.type}||${error.message}`; + const existing = map.get(key); + if (!existing || error.timestamp > existing.timestamp) { + map.set(key, error); + } + } + return Array.from(map.values()).sort( + (a, b) => b.timestamp.getTime() - a.timestamp.getTime(), + ); +} + function mapGroupedHistories( groupedHistories: GroupedAlertHistory[], ): Omit[] { - return groupedHistories.map(group => ({ - createdAt: group._id, - state: groupStateToOverallState(group.states), - counts: group.counts, - lastValues: group.lastValues - .flat() - .sort((a, b) => a.startTime.getTime() - b.startTime.getTime()), - })); + return groupedHistories.map(group => { + // $push skips documents where the field is missing, but be defensive + // about nulls in case of engine differences (e.g. DocumentDB). + const errors = dedupeErrors( + (group.errors ?? []).flat().filter((e): e is IAlertError => e != null), + ); + return { + createdAt: group._id, + state: groupStateToOverallState(group.states), + counts: group.counts, + lastValues: group.lastValues + .flat() + .sort((a, b) => a.startTime.getTime() - b.startTime.getTime()), + ...(errors.length > 0 && { errors }), + }; + }); +} + +/** + * Fetch grouped evaluation windows (one entry per createdAt, newest first) + * for the given alert within the createdAt bounds, capped at `limit` groups. + */ +async function fetchGroupedWindows( + alertId: ObjectId, + createdAt: Record, + limit: number, +): Promise[]> { + const groupedHistories = await AlertHistory.aggregate([ + { + $match: { + alert: new ObjectId(alertId), + createdAt, + }, + }, + { $sort: { createdAt: -1 } }, + { + $group: { + _id: '$createdAt', + states: { + $push: '$state', + }, + counts: { + $sum: '$counts', + }, + lastValues: { + $push: '$lastValues', + }, + errors: { + $push: '$errors', + }, + }, + }, + { + $sort: { + _id: -1, + }, + }, + { + $limit: limit, + }, + ]); + + return mapGroupedHistories(groupedHistories); } /** * Gets the most recent alert histories for a given alert ID, - * limiting to the given number of entries. + * limiting to the given number of entries. Results are one entry per + * evaluation window (grouped by createdAt), newest first. */ export async function getRecentAlertHistories({ alertId, @@ -57,27 +149,189 @@ export async function getRecentAlertHistories({ interval: AlertInterval; limit: number; }): Promise[]> { - const lookbackMs = limit * ALERT_INTERVAL_TO_MINUTES[interval] * 60 * 1000; + // One extra interval of slack so a window sitting exactly `limit` intervals + // back (the newest window is up to one interval old) isn't cut off by the + // lookback bound. + const lookbackMs = (limit + 1) * intervalToMs(interval); + return fetchGroupedWindows( + alertId, + { $gte: new Date(Date.now() - lookbackMs) }, + limit, + ); +} - const groupedHistories = await AlertHistory.aggregate([ +type AlertEvaluationGroupEntry = { + group: string; + state: AlertState; + counts: number; + /** The group's most recent bucket value in this window, if any. */ + lastValue?: { startTime: Date; count: number }; + /** True when a notification was actually sent for this group. */ + fired?: boolean; +}; + +type AlertEvaluationEntry = Omit & { + /** + * Per-group breakdown for group-by alerts, firing-first, capped at + * ALERT_EVALUATION_GROUPS_LIMIT. Absent for non-grouped alerts. + */ + groups?: AlertEvaluationGroupEntry[]; + /** Total number of groups evaluated in this window (before the cap). */ + groupsTotal?: number; +}; + +export type AlertEvaluationsPage = { + data: AlertEvaluationEntry[]; + /** True when older windows may exist within [startTime, ...). */ + hasMore: boolean; + /** Cursor for the next-older page (pass as `before`). Set when hasMore. */ + nextBefore?: Date; +}; + +// One AlertHistory row's fields, as pushed into a per-window sub-array by the +// evaluations aggregation (unlike the alerts-page pipeline, group identity is +// preserved so windows can be broken down per group). +type EvaluationWindowRow = { + group?: string; + state: AlertState; + counts?: number; + lastValues?: IAlertHistory['lastValues']; + fired?: boolean; + errors?: IAlertError[]; + analytics?: IAlertHistoryAnalytics; +}; + +type StructuredWindow = { + _id: Date; + rows: EvaluationWindowRow[]; +}; + +// Firing-first ordering for the per-group breakdown. +const groupStatePriority = (state: AlertState): number => { + switch (state) { + case AlertState.ALERT: + return 0; + case AlertState.PENDING: + return 1; + default: + return 2; + } +}; + +/** + * Pick the evaluation analytics for a window. A window's rows all carry the + * same evaluation-level analytics, except when it contains rows from two + * evaluations (a failed attempt's ERROR row + the successful retry's rows) — + * prefer the successful evaluation's. For rows written before analytics + * existed, `backfilledBuckets` is derived from the distinct bucket times so + * the "Backfilled Buckets" column works on historical data. + */ +function resolveWindowAnalytics( + rows: EvaluationWindowRow[], + lastValues: IAlertHistory['lastValues'], +): IAlertHistoryAnalytics | undefined { + const analytics = + rows.find(r => r.state !== AlertState.ERROR && r.analytics != null) + ?.analytics ?? rows.find(r => r.analytics != null)?.analytics; + + if (analytics?.backfilledBuckets != null) { + return analytics; + } + + const distinctBuckets = new Set(lastValues.map(v => v.startTime.getTime())) + .size; + const derivedBackfilled = Math.max(0, distinctBuckets - 1); + if (analytics == null && derivedBackfilled === 0) { + return undefined; + } + return { ...analytics, backfilledBuckets: derivedBackfilled }; +} + +function mapStructuredWindow(window: StructuredWindow): AlertEvaluationEntry { + const rows = window.rows ?? []; + const errors = dedupeErrors( + rows.flatMap(r => r.errors ?? []).filter(e => e != null), + ); + const lastValues = rows + .flatMap(r => r.lastValues ?? []) + .sort((a, b) => a.startTime.getTime() - b.startTime.getTime()); + const analytics = resolveWindowAnalytics(rows, lastValues); + + // Per-group breakdown: rows carrying a group identity. ERROR rows never + // carry one (they record the evaluation failure, not a group result). + const groupRows = rows.filter( + r => r.group != null && r.group !== '' && r.state !== AlertState.ERROR, + ); + const groups = groupRows + .map(r => { + const groupLastValues = r.lastValues ?? []; + const lastValue = + groupLastValues.length > 0 + ? groupLastValues.reduce((latest, v) => + v.startTime.getTime() > latest.startTime.getTime() ? v : latest, + ) + : undefined; + return { + group: r.group as string, + state: r.state, + counts: r.counts ?? 0, + ...(lastValue != null && { lastValue }), + ...(r.fired != null && { fired: r.fired }), + }; + }) + .sort( + (a, b) => + groupStatePriority(a.state) - groupStatePriority(b.state) || + (b.lastValue?.count ?? Number.NEGATIVE_INFINITY) - + (a.lastValue?.count ?? Number.NEGATIVE_INFINITY) || + a.group.localeCompare(b.group), + ); + + return { + createdAt: window._id, + state: groupStateToOverallState(rows.map(r => r.state)), + counts: rows.reduce((sum, r) => sum + (r.counts ?? 0), 0), + lastValues, + ...(errors.length > 0 && { errors }), + ...(analytics != null && { analytics }), + ...(groups.length > 0 && { + groups: groups.slice(0, ALERT_EVALUATION_GROUPS_LIMIT), + groupsTotal: groups.length, + }), + }; +} + +/** + * Fetch evaluation windows (one entry per createdAt, newest first) within the + * createdAt bounds, capped at `limit` windows, preserving per-group rows so + * grouped alerts can be broken down per group. + */ +async function fetchStructuredWindows( + alertId: ObjectId, + createdAt: Record, + limit: number, +): Promise { + const windows = await AlertHistory.aggregate([ { $match: { alert: new ObjectId(alertId), - createdAt: { $gte: new Date(Date.now() - lookbackMs) }, + createdAt, }, }, { $sort: { createdAt: -1 } }, { $group: { _id: '$createdAt', - states: { - $push: '$state', - }, - counts: { - $sum: '$counts', - }, - lastValues: { - $push: '$lastValues', + rows: { + $push: { + group: '$group', + state: '$state', + counts: '$counts', + lastValues: '$lastValues', + fired: '$fired', + errors: '$errors', + analytics: '$analytics', + }, }, }, }, @@ -91,7 +345,83 @@ export async function getRecentAlertHistories({ }, ]); - return mapGroupedHistories(groupedHistories); + return windows.map(mapStructuredWindow); +} + +/** + * Paginated evaluation windows for the alert detail page, newest first, + * scoped to [startTime, endTime]. + * + * Every request scans a hard-bounded slice of at most ~(limit + 1) intervals + * of history (anchored at `before ?? endTime`), so a wide time range can + * never force an unbounded scan — group-by alerts can have many rows per + * window, and the $group stage processes every matched row. + * + * Because of that bound, a page may end before reaching `startTime` even if + * fewer than `limit` windows were returned (a gap with no evaluations). The + * returned `nextBefore` cursor always advances past the scanned slice, so + * callers can keep paging across gaps: pass it as `before` on the next call. + */ +export async function getAlertEvaluations({ + alertId, + interval, + limit, + startTime, + endTime, + before, +}: { + alertId: ObjectId; + interval: AlertInterval; + limit: number; + startTime: Date; + endTime: Date; + before?: Date; +}): Promise { + const intervalMs = intervalToMs(interval); + // One extra interval of slack so a window sitting exactly `limit` intervals + // back isn't cut off by the scan bound. + const scanMs = (limit + 1) * intervalMs; + + // Upper bound: the page cursor when provided (exclusive), else the range + // end (inclusive). A cursor past the range end is ignored. + const usableBefore = + before != null && before.getTime() <= endTime.getTime() + ? before + : undefined; + const pageEndMs = usableBefore?.getTime() ?? endTime.getTime(); + + // Lower bound: the scan bound, clamped to the requested range start. + const scanFloorMs = Math.max(startTime.getTime(), pageEndMs - scanMs); + + const createdAt: Record = { + $gte: new Date(scanFloorMs), + }; + if (usableBefore != null) { + createdAt.$lt = usableBefore; + } else { + createdAt.$lte = endTime; + } + + // Fetch one extra window to detect count truncation. + const windows = await fetchStructuredWindows(alertId, createdAt, limit + 1); + + const truncatedByCount = windows.length > limit; + const data = truncatedByCount ? windows.slice(0, limit) : windows; + // More windows may exist when the page filled up, or when the scan bound + // stopped before reaching the range start. + const truncatedByScanBound = scanFloorMs > startTime.getTime(); + const hasMore = truncatedByCount || truncatedByScanBound; + + // Count truncation: resume strictly before the last returned window. + // Scan-bound truncation: the slice down to scanFloor (inclusive) is fully + // covered, so resume strictly before it. + const nextBefore = !hasMore + ? undefined + : truncatedByCount + ? data[data.length - 1].createdAt + : new Date(scanFloorMs); + + return { data, hasMore, nextBefore }; } /** @@ -150,16 +480,19 @@ export async function getAlertTransitionsInRange({ startTime: Date; endTime: Date; }): Promise { - const intervalMs = ALERT_INTERVAL_TO_MINUTES[interval] * 60 * 1000; + const intervalMs = intervalToMs(interval); const lookbackStart = new Date(startTime.getTime() - intervalMs); - // Only the per-window state is needed to detect crossings. + // Only the per-window state is needed to detect crossings. ERROR rows are + // failed evaluations, not state observations — excluding them prevents a + // query failure mid-firing from drawing a false recovery annotation. const windows = await AlertHistory.aggregate<{ _id: Date; states: string[] }>( [ { $match: { alert: new ObjectId(alertId), createdAt: { $gte: lookbackStart, $lte: endTime }, + state: { $ne: AlertState.ERROR }, }, }, { $group: { _id: '$createdAt', states: { $push: '$state' } } }, diff --git a/packages/api/src/models/alert.ts b/packages/api/src/models/alert.ts index 93dfbc42c5..fd5e1df9e2 100644 --- a/packages/api/src/models/alert.ts +++ b/packages/api/src/models/alert.ts @@ -12,6 +12,13 @@ import Team from './team'; export enum AlertState { ALERT = 'ALERT', DISABLED = 'DISABLED', + /** + * Only used on AlertHistory records (never on the alert itself): marks an + * evaluation window whose evaluation or notification failed. ERROR history + * rows are excluded from alert scheduling/backfill computations so the + * failed window is still retried. + */ + ERROR = 'ERROR', INSUFFICIENT_DATA = 'INSUFFICIENT_DATA', OK = 'OK', PENDING = 'PENDING', diff --git a/packages/api/src/models/alertHistory.ts b/packages/api/src/models/alertHistory.ts index 5cf8630f81..78b968208c 100644 --- a/packages/api/src/models/alertHistory.ts +++ b/packages/api/src/models/alertHistory.ts @@ -1,10 +1,35 @@ +import { AlertErrorType } from '@hyperdx/common-utils/dist/types'; import mongoose, { Schema } from 'mongoose'; import ms from 'ms'; -import { AlertState } from '@/models/alert'; +import { AlertState, IAlertError } from '@/models/alert'; import type { ObjectId } from '.'; +/** + * Diagnostics for the evaluation that wrote a history record. + * Evaluation-level: identical on every row one evaluation writes (including + * per-group rows). + */ +export interface IAlertHistoryAnalytics { + /** + * ClickHouse query duration for the evaluation (ms). On query-failure + * ERROR records, the time until the query failed — for QUERY_TIMEOUT this + * is approximately the configured evaluation timeout. + */ + queryDurationMs?: number; + /** + * Total wall time delivering webhook notifications in the evaluation, + * including retries (ms). + */ + webhookDurationMs?: number; + /** + * Earlier buckets backfilled in this run after missed ticks + * (expected buckets − 1). 0 in steady state. + */ + backfilledBuckets?: number; +} + export interface IAlertHistory { alert: ObjectId; counts: number; @@ -13,6 +38,14 @@ export interface IAlertHistory { lastValues: { startTime: Date; count: number }[]; group?: string; // For group-by alerts, stores the group identifier fired?: boolean; + /** + * Errors recorded for this evaluation window. Present on ERROR-state rows + * (query/processing failures where no normal history is written) and on + * the ERROR row created alongside normal rows when notifications fail. + */ + errors?: IAlertError[]; + /** Diagnostics for the evaluation that wrote this record. */ + analytics?: IAlertHistoryAnalytics; } const AlertHistorySchema = new Schema({ @@ -50,6 +83,32 @@ const AlertHistorySchema = new Schema({ type: Boolean, required: false, }, + errors: { + type: [ + { + _id: false, + timestamp: { type: Date, required: true }, + type: { + type: String, + enum: AlertErrorType, + required: true, + }, + message: { type: String, required: true }, + }, + ], + required: false, + default: undefined, + }, + analytics: { + type: { + _id: false, + queryDurationMs: { type: Number, required: false }, + webhookDurationMs: { type: Number, required: false }, + backfilledBuckets: { type: Number, required: false }, + }, + required: false, + default: undefined, + }, }); AlertHistorySchema.index( diff --git a/packages/api/src/routers/api/__tests__/alerts.int.test.ts b/packages/api/src/routers/api/__tests__/alerts.int.test.ts index 01dce61b1b..6250c0203a 100644 --- a/packages/api/src/routers/api/__tests__/alerts.int.test.ts +++ b/packages/api/src/routers/api/__tests__/alerts.int.test.ts @@ -1132,6 +1132,232 @@ describe('alerts router', () => { }); }); + describe('GET /alerts/:id/evaluations', () => { + const createTileAlert = async (): Promise => { + const dashboard = await agent + .post('/dashboards') + .send(MOCK_DASHBOARD) + .expect(200); + const alert = await agent + .post('/alerts') + .send( + makeAlertInput({ + dashboardId: dashboard.body.id, + tileId: dashboard.body.tiles[0].id, + webhookId: webhook._id.toString(), + }), + ) + .expect(200); + return String(alert.body.data._id); + }; + + it('returns evaluation windows newest-first, including error windows', async () => { + const alertId = await createTileAlert(); + const now = Date.now(); + const at = (minsAgo: number) => new Date(now - minsAgo * 60_000); + + await AlertHistory.create({ + alert: alertId, + createdAt: at(10), + state: AlertState.OK, + counts: 0, + lastValues: [{ startTime: at(10), count: 0 }], + }); + await AlertHistory.create({ + alert: alertId, + createdAt: at(5), + state: AlertState.ERROR, + counts: 0, + lastValues: [], + errors: [ + { + timestamp: at(4), + type: AlertErrorType.QUERY_TIMEOUT, + message: 'Alert query did not complete within the 300s timeout', + }, + ], + }); + + const res = await agent + .get(`/alerts/${alertId}/evaluations`) + .query({ startTime: at(30).getTime(), endTime: now }) + .expect(200); + + expect(res.body.hasMore).toBe(false); + expect(res.body.data).toHaveLength(2); + expect(res.body.data[0].state).toBe(AlertState.ERROR); + expect(res.body.data[0].createdAt).toBe(at(5).toISOString()); + expect(res.body.data[0].errors).toHaveLength(1); + expect(res.body.data[0].errors[0].type).toBe( + AlertErrorType.QUERY_TIMEOUT, + ); + expect(res.body.data[1].state).toBe(AlertState.OK); + }); + + it('paginates with limit + the nextBefore cursor and reports hasMore', async () => { + const alertId = await createTileAlert(); + const now = Date.now(); + // Windows aligned to the alert interval cadence (5m apart) + const windows = [5, 10, 15].map( + minsAgo => new Date(now - minsAgo * 60_000), + ); + for (const createdAt of windows) { + await AlertHistory.create({ + alert: alertId, + createdAt, + state: AlertState.OK, + counts: 0, + lastValues: [{ startTime: createdAt, count: 0 }], + }); + } + + // Range chosen so the second page's bounded scan reaches startTime + // exactly (limit=2 → each page scans (2+1)×5m = 15m of history). + const startTime = now - 20 * 60_000; + const endTime = now; + + const firstPage = await agent + .get(`/alerts/${alertId}/evaluations`) + .query({ limit: 2, startTime, endTime }) + .expect(200); + expect(firstPage.body.data).toHaveLength(2); + expect(firstPage.body.hasMore).toBe(true); + expect(firstPage.body.nextBefore).toBe(windows[1].getTime()); + expect(firstPage.body.data[0].createdAt).toBe(windows[0].toISOString()); + + const secondPage = await agent + .get(`/alerts/${alertId}/evaluations`) + .query({ + limit: 2, + startTime, + endTime, + before: firstPage.body.nextBefore, + }) + .expect(200); + expect(secondPage.body.data).toHaveLength(1); + expect(secondPage.body.hasMore).toBe(false); + expect(secondPage.body.nextBefore).toBeUndefined(); + expect(secondPage.body.data[0].createdAt).toBe(windows[2].toISOString()); + }); + + it('returns the per-group breakdown for grouped windows', async () => { + const alertId = await createTileAlert(); + const now = Date.now(); + const windowStart = new Date(now - 5 * 60_000); + const bucket = new Date(now - 10 * 60_000); + + await AlertHistory.create({ + alert: alertId, + createdAt: windowStart, + state: AlertState.ALERT, + counts: 2, + lastValues: [{ startTime: bucket, count: 12 }], + group: 'ServiceName:api', + fired: true, + }); + await AlertHistory.create({ + alert: alertId, + createdAt: windowStart, + state: AlertState.OK, + counts: 0, + lastValues: [{ startTime: bucket, count: 1 }], + group: 'ServiceName:web', + }); + + const res = await agent + .get(`/alerts/${alertId}/evaluations`) + .query({ startTime: now - 30 * 60_000, endTime: now }) + .expect(200); + + expect(res.body.data).toHaveLength(1); + const window = res.body.data[0]; + expect(window.state).toBe(AlertState.ALERT); + expect(window.groupsTotal).toBe(2); + expect(window.groups).toHaveLength(2); + // Firing group first + expect(window.groups[0]).toMatchObject({ + group: 'ServiceName:api', + state: AlertState.ALERT, + counts: 2, + fired: true, + }); + expect(window.groups[0].lastValue.count).toBe(12); + expect(window.groups[1]).toMatchObject({ + group: 'ServiceName:web', + state: AlertState.OK, + }); + }); + + it('scopes results to the requested time range', async () => { + const alertId = await createTileAlert(); + const now = Date.now(); + const at = (minsAgo: number) => new Date(now - minsAgo * 60_000); + + await AlertHistory.create({ + alert: alertId, + createdAt: at(5), + state: AlertState.OK, + counts: 0, + lastValues: [{ startTime: at(5), count: 0 }], + }); + await AlertHistory.create({ + alert: alertId, + createdAt: at(45), + state: AlertState.OK, + counts: 0, + lastValues: [{ startTime: at(45), count: 0 }], + }); + + const res = await agent + .get(`/alerts/${alertId}/evaluations`) + .query({ startTime: at(30).getTime(), endTime: now }) + .expect(200); + + expect(res.body.data).toHaveLength(1); + expect(res.body.data[0].createdAt).toBe(at(5).toISOString()); + expect(res.body.hasMore).toBe(false); + }); + + it('accepts a very wide range (span is clamped, not rejected)', async () => { + const alertId = await createTileAlert(); + await agent + .get(`/alerts/${alertId}/evaluations`) + .query({ startTime: 1, endTime: Date.now() }) + .expect(200); + }); + + it('rejects an out-of-range limit', async () => { + const alertId = await createTileAlert(); + await agent + .get(`/alerts/${alertId}/evaluations`) + .query({ limit: 10_000 }) + .expect(400); + }); + + it('rejects startTime >= endTime', async () => { + const alertId = await createTileAlert(); + const now = Date.now(); + await agent + .get(`/alerts/${alertId}/evaluations`) + .query({ startTime: now, endTime: now - 60_000 }) + .expect(400); + }); + + it('returns 404 for an unknown alert id', async () => { + await agent.get(`/alerts/${randomMongoId()}/evaluations`).expect(404); + }); + + it("returns 404 for another team's alert", async () => { + const otherTeamAlert = await Alert.create({ + team: randomMongoId(), + threshold: 1, + interval: '5m', + channel: { type: null }, + }); + await agent.get(`/alerts/${otherTeamAlert._id}/evaluations`).expect(404); + }); + }); + describe('errors propagation', () => { it('returns the errors field on a single alert response', async () => { const dashboard = await agent diff --git a/packages/api/src/routers/api/alerts.ts b/packages/api/src/routers/api/alerts.ts index 915be23272..a44603c5a0 100644 --- a/packages/api/src/routers/api/alerts.ts +++ b/packages/api/src/routers/api/alerts.ts @@ -1,5 +1,6 @@ import type { AlertApiResponse, + AlertEvaluationsApiResponse, AlertHistoryRangeApiResponse, AlertsApiResponse, AlertsPageItem, @@ -11,6 +12,7 @@ import { z } from 'zod'; import { processRequest, validateRequest } from 'zod-express-middleware'; import { + getAlertEvaluations, getAlertTransitionsInRange, getRecentAlertHistories, getRecentAlertHistoriesBatch, @@ -88,6 +90,7 @@ const formatAlertResponse = ( 'numConsecutiveWindows', ]), tileId: alert.tileId ?? undefined, + groupBy: alert.groupBy ?? undefined, }; }; @@ -155,11 +158,93 @@ router.get( }, ); +// Alert history has a ~30-day TTL, so cap queried spans to bound the +// aggregations regardless of how small a startTime the caller sends. +const MAX_HISTORY_SPAN_MS = 31 * 24 * 60 * 60 * 1000; + +// Paginated evaluation history for the alert detail page: one entry per +// evaluation window (grouped across group-by groups), newest first, including +// any errors recorded for the window. Scoped to [startTime, endTime] (epoch +// ms; endTime defaults to now, startTime is clamped to the history retention +// span). `before` (epoch ms, from the previous response's `nextBefore`) pages +// to older windows within the range. Note: /:id/history (below) returns +// firing transitions for chart annotations, which is a different shape. +const EVALUATIONS_LIMIT = 200; +type AlertEvaluationsExpRes = express.Response; +router.get( + '/:id/evaluations', + processRequest({ + params: z.object({ id: objectIdSchema }), + query: z + .object({ + limit: z.coerce + .number() + .int() + .min(1) + .max(EVALUATIONS_LIMIT) + .default(EVALUATIONS_LIMIT), + before: z.coerce.number().int().positive().optional(), + startTime: z.coerce.number().int().positive().optional(), + endTime: z.coerce.number().int().positive().optional(), + }) + .refine( + q => + q.startTime == null || q.endTime == null || q.startTime < q.endTime, + { message: 'startTime must be less than endTime' }, + ), + }), + async (req, res: AlertEvaluationsExpRes, next) => { + try { + const teamId = req.user?.team; + if (teamId == null) { + return res.sendStatus(403); + } + + // Scope to the caller's team (404 for alerts they can't see). + const alert = await getAlertById(req.params.id, teamId); + if (!alert) { + return res.sendStatus(404); + } + + // zod applies the default at runtime, but the middleware types the + // parsed query with the input (pre-default) shape. + const limit = req.query.limit ?? EVALUATIONS_LIMIT; + const { before } = req.query; + const endTime = + req.query.endTime != null ? new Date(req.query.endTime) : new Date(); + // Clamp the span so a tiny/zero startTime can't page beyond the history + // retention window (same cap as the /history transitions endpoint). + const startTime = new Date( + Math.max( + req.query.startTime ?? endTime.getTime() - MAX_HISTORY_SPAN_MS, + endTime.getTime() - MAX_HISTORY_SPAN_MS, + ), + ); + + const page = await getAlertEvaluations({ + alertId: new ObjectId(alert._id), + interval: alert.interval, + limit, + startTime, + endTime, + before: before != null ? new Date(before) : undefined, + }); + + sendJson(res, { + data: page.data, + hasMore: page.hasMore, + ...(page.nextBefore != null && { + nextBefore: page.nextBefore.getTime(), + }), + }); + } catch (e) { + next(e); + } + }, +); + // Alert firing/recovery transitions within a time range, used to draw // annotations on dashboard charts (startTime/endTime are epoch milliseconds). -// Alert history has a ~30-day TTL, so cap the queried span to bound the -// aggregation regardless of how small a startTime the caller sends. -const MAX_HISTORY_SPAN_MS = 31 * 24 * 60 * 60 * 1000; type AlertHistoryRangeExpRes = express.Response; router.get( '/:id/history', diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 154da9d24a..c56e6ae453 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -597,6 +597,13 @@ export const isRangeThresholdType = (type: string): boolean => export enum AlertState { ALERT = 'ALERT', DISABLED = 'DISABLED', + /** + * Only used on AlertHistory records (never on the alert itself): marks an + * evaluation window whose evaluation or notification failed. ERROR history + * rows are excluded from alert scheduling/backfill computations so the + * failed window is still retried. + */ + ERROR = 'ERROR', INSUFFICIENT_DATA = 'INSUFFICIENT_DATA', OK = 'OK', PENDING = 'PENDING', @@ -604,6 +611,8 @@ export enum AlertState { export enum AlertErrorType { QUERY_ERROR = 'QUERY_ERROR', + /** The alert query did not complete within the evaluation timeout. */ + QUERY_TIMEOUT = 'QUERY_TIMEOUT', WEBHOOK_ERROR = 'WEBHOOK_ERROR', INVALID_ALERT = 'INVALID_ALERT', UNKNOWN = 'UNKNOWN', @@ -799,15 +808,40 @@ export const AlertSchema = z.union([ export type Alert = z.infer; +// Diagnostics for the evaluation that wrote a history record. Evaluation- +// level: identical on every row one evaluation writes (incl. per-group rows). +export const AlertHistoryAnalyticsSchema = z.object({ + /** ClickHouse query duration for the evaluation (ms). On query-failure ERROR records, the time until the query failed. */ + queryDurationMs: z.number().optional(), + /** Total wall time delivering webhook notifications in the evaluation, including retries (ms). */ + webhookDurationMs: z.number().optional(), + /** Earlier buckets backfilled in this run after missed ticks (expected buckets − 1). */ + backfilledBuckets: z.number().optional(), +}); + +export type AlertHistoryAnalytics = z.infer; + export const AlertHistorySchema = z.object({ counts: z.number(), createdAt: z.string(), lastValues: z.array(z.object({ startTime: z.string(), count: z.number() })), state: z.nativeEnum(AlertState), + /** Errors recorded for this evaluation window (query/webhook failures). */ + errors: z.array(AlertErrorSchema).optional(), + /** Diagnostics for the evaluation that wrote this record. */ + analytics: AlertHistoryAnalyticsSchema.optional(), }); export type AlertHistory = z.infer; +/** + * Max per-group entries returned per evaluation window on the alert detail + * page. Groups are sorted firing-first before the cap, so firing groups are + * always visible. Shared between the API (enforces the cap) and the app + * (explains it in the UI). + */ +export const ALERT_EVALUATION_GROUPS_LIMIT = 50; + // A single alert state transition within a time range, used to draw // firing/recovery annotations on dashboard charts. Only boundary crossings are // emitted: ALERT = fired, OK = recovered. @@ -2103,6 +2137,7 @@ export const AlertsPageItemSchema = z.object({ dashboardId: z.string().optional(), savedSearchId: z.string().optional(), tileId: z.string().optional(), + groupBy: z.string().optional(), name: z.string().nullish(), message: z.string().nullish(), note: alertNoteSchema, @@ -2171,6 +2206,50 @@ export type AlertHistoryRangeApiResponse = z.infer< typeof AlertHistoryRangeApiResponseSchema >; +// Per-group result of one evaluation window for a group-by alert. +export const AlertEvaluationGroupSchema = z.object({ + group: z.string(), + state: z.nativeEnum(AlertState), + counts: z.number(), + /** The group's most recent bucket value in this window, if any. */ + lastValue: z.object({ startTime: z.string(), count: z.number() }).optional(), + /** True when a notification was actually sent for this group. */ + fired: z.boolean().optional(), +}); + +export type AlertEvaluationGroup = z.infer; + +// One evaluation window on the alert detail page. For group-by alerts, +// carries the per-group breakdown (firing-first, capped server-side). +export const AlertEvaluationSchema = AlertHistorySchema.extend({ + groups: z.array(AlertEvaluationGroupSchema).optional(), + /** Total number of groups evaluated in this window (before the cap). */ + groupsTotal: z.number().optional(), +}); + +export type AlertEvaluation = z.infer; + +// Paginated evaluation history for the alert detail page. Each entry is one +// evaluation window (newest first), including any errors recorded for it. +export const AlertEvaluationsApiResponseSchema = z.object({ + data: z.array(AlertEvaluationSchema), + /** + * True when older evaluation windows may exist within the requested time + * range beyond the returned page. + */ + hasMore: z.boolean(), + /** + * Cursor (epoch ms) for the next-older page: pass as `before` on the next + * request. Present when hasMore is true. Cursor-based (not offset-based) + * so pages advance even across gaps with no evaluations. + */ + nextBefore: z.number().optional(), +}); + +export type AlertEvaluationsApiResponse = z.infer< + typeof AlertEvaluationsApiResponseSchema +>; + // Webhooks export const WebhooksApiResponseSchema = z.object({ data: z.array(WebhookSchema), From 1c5e0913f4bd84ef47b1351b46f4f9efbefce8cb Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Thu, 6 Aug 2026 15:57:26 -0700 Subject: [PATCH 2/3] feat(app): alert detail page with evaluation history (HDX-4997) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Datadog-style alert status page at /alerts/:id, reachable via a Details link on each alerts-page row (the alert name keeps linking to its saved search / dashboard tile): - Header with state badge, silence/ack, source link, and a time picker; the alert's underlying query charted over the selected range with threshold reference lines and firing/recovery annotations. - Widened evaluation-history strip (60 windows); chart, strip, and event stream all follow the picker's exact range. - Evaluation event stream: one parent row per window labeled with the evaluated bucket start (matching the chart's x-axis), state, latest value, breaches, backfilled buckets, query/webhook durations, and error labels; per-group child rows for group-by alerts; older windows load via an infinite-scroll sentinel using the endpoint's nextBefore cursor. A failed page fetch unmounts the sentinel (whose effect would otherwise refire forever) and renders an explicit retry affordance; a failed initial page shows a failure message instead of the empty state. - Alerts-page history strip renders errored evaluation windows as striped-red segments with full error details in a modal on click. - The Details link and /alerts/:id route are gated behind NEXT_PUBLIC_ENABLE_ALERT_DETAILS (default off) — enabled in dev (.env.development) and CI (e2e webserver) only while the feature bakes; self-hosted deployments opt in via docker-compose. Includes unit tests and full-stack e2e coverage seeded directly in MongoDB. --- docker-compose.yml | 2 + packages/app/.env.development | 1 + packages/app/pages/alerts/[alertId].tsx | 3 + .../pages/{alerts.tsx => alerts/index.tsx} | 0 packages/app/playwright.config.ts | 4 +- packages/app/src/AlertDetailPage.tsx | 286 +++++++++++++ packages/app/src/AlertsPage.tsx | 38 +- packages/app/src/api.ts | 36 +- .../app/src/components/AlertPreviewChart.tsx | 25 +- .../components/alerts/AlertDetailChart.tsx | 275 ++++++++++++ .../alerts/AlertEvaluationsTable.tsx | 396 ++++++++++++++++++ .../components/alerts/AlertHistoryCards.tsx | 194 ++++++--- .../__tests__/AlertEvaluationsTable.test.tsx | 292 +++++++++++++ .../__tests__/AlertHistoryCards.test.tsx | 71 ++++ packages/app/src/config.ts | 4 + packages/app/styles/AlertsPage.module.scss | 18 + .../app/tests/e2e/features/alerts.spec.ts | 50 +++ .../app/tests/e2e/global-setup-fullstack.ts | 35 ++ .../app/tests/e2e/page-objects/AlertsPage.ts | 37 ++ 19 files changed, 1688 insertions(+), 79 deletions(-) create mode 100644 packages/app/pages/alerts/[alertId].tsx rename packages/app/pages/{alerts.tsx => alerts/index.tsx} (100%) create mode 100644 packages/app/src/AlertDetailPage.tsx create mode 100644 packages/app/src/components/alerts/AlertDetailChart.tsx create mode 100644 packages/app/src/components/alerts/AlertEvaluationsTable.tsx create mode 100644 packages/app/src/components/alerts/__tests__/AlertEvaluationsTable.test.tsx create mode 100644 packages/app/src/components/alerts/__tests__/AlertHistoryCards.test.tsx diff --git a/docker-compose.yml b/docker-compose.yml index bdd7339e03..041023c827 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -74,6 +74,8 @@ services: # Uncomment the next two lines to enable PromQL (Prometheus-compatible metrics) # ENABLE_PROMQL: 'true' # NEXT_PUBLIC_ENABLE_PROMQL: 'true' + # Uncomment to enable the alert detail page (in development) + # NEXT_PUBLIC_ENABLE_ALERT_DETAILS: 'true' DEFAULT_CONNECTIONS: '[{"name":"Local ClickHouse","host":"http://ch-server:8123","username":"default","password":""}]' diff --git a/packages/app/.env.development b/packages/app/.env.development index 7fd59cdfde..af4a79ce0f 100644 --- a/packages/app/.env.development +++ b/packages/app/.env.development @@ -13,3 +13,4 @@ NEXT_PUBLIC_OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:${HDX_DEV_OTEL_HTTP_PO # NEXT_PUBLIC_OTEL_EXPORTER_OTLP_METRICS_ENDPOINT= # NEXT_PUBLIC_OTEL_EXPORTER_OTLP_LOGS_ENDPOINT= NEXT_PUBLIC_ENABLE_PROMQL=true +NEXT_PUBLIC_ENABLE_ALERT_DETAILS=true diff --git a/packages/app/pages/alerts/[alertId].tsx b/packages/app/pages/alerts/[alertId].tsx new file mode 100644 index 0000000000..1b0dc50f9a --- /dev/null +++ b/packages/app/pages/alerts/[alertId].tsx @@ -0,0 +1,3 @@ +import AlertDetailPage from '@/AlertDetailPage'; + +export default AlertDetailPage; diff --git a/packages/app/pages/alerts.tsx b/packages/app/pages/alerts/index.tsx similarity index 100% rename from packages/app/pages/alerts.tsx rename to packages/app/pages/alerts/index.tsx diff --git a/packages/app/playwright.config.ts b/packages/app/playwright.config.ts index b6d8f29353..2a73b3c344 100644 --- a/packages/app/playwright.config.ts +++ b/packages/app/playwright.config.ts @@ -105,8 +105,8 @@ export default defineConfig({ { // Full UI: Alerts + Dashboards. Not local mode; Alerts enabled; command: USE_DEV - ? `SERVER_URL=http://localhost:${API_PORT} PORT=${APP_PORT} NEXT_DIST_DIR=.next-e2e next dev --webpack` - : `SERVER_URL=http://localhost:${API_PORT} PORT=${APP_PORT} NEXT_DIST_DIR=.next-e2e yarn build && SERVER_URL=http://localhost:${API_PORT} PORT=${APP_PORT} NEXT_DIST_DIR=.next-e2e yarn start`, + ? `SERVER_URL=http://localhost:${API_PORT} PORT=${APP_PORT} NEXT_PUBLIC_ENABLE_ALERT_DETAILS=true NEXT_DIST_DIR=.next-e2e next dev --webpack` + : `SERVER_URL=http://localhost:${API_PORT} PORT=${APP_PORT} NEXT_PUBLIC_ENABLE_ALERT_DETAILS=true NEXT_DIST_DIR=.next-e2e yarn build && SERVER_URL=http://localhost:${API_PORT} PORT=${APP_PORT} NEXT_PUBLIC_ENABLE_ALERT_DETAILS=true NEXT_DIST_DIR=.next-e2e yarn start`, port: parseInt(APP_PORT, 10), reuseExistingServer: !process.env.CI, timeout: APP_SERVER_STARTUP_TIMEOUT_MS, diff --git a/packages/app/src/AlertDetailPage.tsx b/packages/app/src/AlertDetailPage.tsx new file mode 100644 index 0000000000..d59f2805f8 --- /dev/null +++ b/packages/app/src/AlertDetailPage.tsx @@ -0,0 +1,286 @@ +import * as React from 'react'; +import Head from 'next/head'; +import Link from 'next/link'; +import { useRouter } from 'next/router'; +import { + AlertSource, + isRangeThresholdType, +} from '@hyperdx/common-utils/dist/types'; +import { + Anchor, + Breadcrumbs, + Button, + Container, + Group, + Skeleton, + Stack, + Text, +} from '@mantine/core'; +import { IconExternalLink } from '@tabler/icons-react'; + +import { AckAlert } from '@/components/alerts/AckAlert'; +import { AlertDetailChart } from '@/components/alerts/AlertDetailChart'; +import { + AlertEvaluationsTable, + AlertStateBadge, +} from '@/components/alerts/AlertEvaluationsTable'; +import { AlertHistoryCardList } from '@/components/alerts/AlertHistoryCards'; +import EmptyState from '@/components/EmptyState'; +import { PageHeader } from '@/components/PageHeader'; +import { TimePicker } from '@/components/TimePicker'; +import { IS_ALERT_DETAILS_ENABLED } from '@/config'; + +import { useBrandDisplayName } from './theme/ThemeProvider'; +import { TILE_ALERT_THRESHOLD_TYPE_OPTIONS } from './utils/alerts'; +import { getWebhookChannelIcon } from './utils/webhookIcons'; +import { + AlertNote, + getAlertDisplayName, + getAlertSourceUrl, +} from './AlertsPage'; +import api from './api'; +import { withAppNav } from './layout'; +import { parseTimeQuery, useNewTimeQuery } from './timeQuery'; +import type { AlertsPageItem } from './types'; + +import styles from '@styles/AlertsPage.module.scss'; + +const DEFAULT_TIME_RANGE_LABEL = 'Past 12h'; +// eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- parseTimeQuery always resolves a fixed relative label +const defaultTimeRange = parseTimeQuery(DEFAULT_TIME_RANGE_LABEL, false) as [ + Date, + Date, +]; + +// Number of evaluation windows in the timeline strip — wider than the +// alerts-page strip so failure/firing patterns over time are visible. +const TIMELINE_ITEMS = 60; + +function AlertProperties({ alert }: { alert: AlertsPageItem }) { + const thresholdLabel = + TILE_ALERT_THRESHOLD_TYPE_OPTIONS[alert.thresholdType] ?? + alert.thresholdType; + + return ( + +
+ + If value {thresholdLabel}{' '} + {alert.threshold} + {isRangeThresholdType(alert.thresholdType) && ( + <> + {' '} + and {alert.thresholdMax ?? '-'} + + )} + + · + Evaluates every {alert.interval} + {alert.numConsecutiveWindows != null && + alert.numConsecutiveWindows > 1 && ( + <> + · + + Fires after {alert.numConsecutiveWindows} consecutive windows + + + )} + · + + Notify via {getWebhookChannelIcon(alert.channel.type)} Webhook + + {alert.createdBy && ( + <> + · + + Created by {alert.createdBy.name || alert.createdBy.email} + + + )} +
+ {alert.note && } + + ); +} + +function AlertDetailBody({ alert }: { alert: AlertsPageItem }) { + const alertUrl = getAlertSourceUrl(alert); + + const [displayedTimeInputValue, setDisplayedTimeInputValue] = React.useState( + DEFAULT_TIME_RANGE_LABEL, + ); + const { searchedTimeRange, onSearch } = useNewTimeQuery({ + initialDisplayValue: DEFAULT_TIME_RANGE_LABEL, + initialTimeRange: defaultTimeRange, + setDisplayedTimeInputValue, + }); + + // The chart, timeline strip, and event stream all reflect the exact picked + // time range. Evaluations are fetched in fixed-size pages as the user + // scrolls (each page is a hard-bounded scan server-side, so wide ranges + // never fetch unbounded history). + const { + data: evaluationsData, + isLoading: isEvaluationsLoading, + isError: isEvaluationsError, + hasNextPage, + isFetchingNextPage, + fetchNextPage, + } = api.useAlertEvaluations(alert._id, searchedTimeRange); + + const evaluations = React.useMemo( + () => evaluationsData?.pages.flatMap(page => page.data) ?? [], + [evaluationsData], + ); + + // Stable callback so the scroll sentinel's effect doesn't refire on every + // render. cancelRefetch:false makes overlapping triggers no-ops instead of + // restarting an in-flight page fetch. + const onLoadMore = React.useCallback(() => { + fetchNextPage({ cancelRefetch: false }); + }, [fetchNextPage]); + + return ( + <> + + + Alerts + + + {getAlertDisplayName(alert) || 'Alert'} + + + } + leading={ + + {alert.state != null && } + {getAlertDisplayName(alert)} + + } + actions={ + + + {alertUrl && ( + + )} + + + } + /> +
+ + + + +
+ + Evaluation History + + + +
+ + +
+ + ); +} + +export default function AlertDetailPage() { + const brandName = useBrandDisplayName(); + const router = useRouter(); + const alertId = + typeof router.query.alertId === 'string' ? router.query.alertId : undefined; + + // Direct-URL guard while the feature bakes: the alerts page only renders + // Details links when the flag is on, but the route itself must bounce too. + React.useEffect(() => { + if (!IS_ALERT_DETAILS_ENABLED) { + router.replace('/alerts'); + } + }, [router]); + + const { data, isLoading, isError } = api.useAlert(alertId); + const alert = data?.data; + + if (!IS_ALERT_DETAILS_ENABLED) { + return null; + } + + return ( +
+ + + {alert ? `${getAlertDisplayName(alert)} - Alerts` : 'Alerts'} -{' '} + {brandName} + + + {isLoading && ( + + + + + + + + )} + {!isLoading && (isError || !alert) && ( + + + Back to alerts + + } + /> + + )} + {!isLoading && alert && } +
+ ); +} + +AlertDetailPage.getLayout = withAppNav; diff --git a/packages/app/src/AlertsPage.tsx b/packages/app/src/AlertsPage.tsx index 87e1e6aa16..8edbe3ded2 100644 --- a/packages/app/src/AlertsPage.tsx +++ b/packages/app/src/AlertsPage.tsx @@ -13,6 +13,7 @@ import { Alert, Anchor, Badge, + Button, Collapse, Container, Flex, @@ -43,6 +44,7 @@ import { AlertHistoryCardList } from '@/components/alerts/AlertHistoryCards'; import EmptyState from '@/components/EmptyState'; import ResourceTerraformPopover from '@/components/Iac/ResourceTerraformPopover'; import { PageHeader } from '@/components/PageHeader'; +import { IS_ALERT_DETAILS_ENABLED } from '@/config'; import { useBrandDisplayName } from './theme/ThemeProvider'; import { TILE_ALERT_THRESHOLD_TYPE_OPTIONS } from './utils/alerts'; @@ -53,7 +55,7 @@ import type { AlertsPageItem } from './types'; import styles from '@styles/AlertsPage.module.scss'; -function getAlertDisplayName(alert: AlertsPageItem): string { +export function getAlertDisplayName(alert: AlertsPageItem): string { if (alert.source === AlertSource.TILE && alert.dashboard) { const tile = alert.dashboard.tiles.find(t => t.id === alert.tileId); const tileName = tile?.config.name || 'Tile'; @@ -65,6 +67,17 @@ function getAlertDisplayName(alert: AlertsPageItem): string { return ''; } +/** URL of the saved search / dashboard tile the alert is watching. */ +export function getAlertSourceUrl(alert: AlertsPageItem): string { + if (alert.source === AlertSource.TILE && alert.dashboard) { + return `/dashboards/${alert.dashboardId}?highlightedTileId=${alert.tileId}`; + } + if (alert.source === AlertSource.SAVED_SEARCH && alert.savedSearch) { + return `/search/${alert.savedSearchId}`; + } + return ''; +} + function getAlertTags(alert: AlertsPageItem): string[] { return alert.dashboard?.tags ?? alert.savedSearch?.tags ?? []; } @@ -74,7 +87,7 @@ function getAlertCreatorLabel(alert: AlertsPageItem): string | undefined { return alert.createdBy.name || alert.createdBy.email; } -function AlertNote({ note }: { note: string }) { +export function AlertNote({ note }: { note: string }) { const [opened, { toggle }] = useDisclosure(false); return ( @@ -149,15 +162,7 @@ function AlertDetails({ alert }: { alert: AlertsPageItem }) { return '–'; }, [alert]); - const alertUrl = React.useMemo(() => { - if (alert.source === AlertSource.TILE && alert.dashboard) { - return `/dashboards/${alert.dashboardId}?highlightedTileId=${alert.tileId}`; - } - if (alert.source === AlertSource.SAVED_SEARCH && alert.savedSearch) { - return `/search/${alert.savedSearchId}`; - } - return ''; - }, [alert]); + const alertUrl = React.useMemo(() => getAlertSourceUrl(alert), [alert]); const alertIcon = (() => { switch (alert.source) { @@ -286,6 +291,17 @@ function AlertDetails({ alert }: { alert: AlertsPageItem }) { )} + {IS_ALERT_DETAILS_ENABLED && ( + + )} ); diff --git a/packages/app/src/api.ts b/packages/app/src/api.ts index 21facd9c05..a8dea6a600 100644 --- a/packages/app/src/api.ts +++ b/packages/app/src/api.ts @@ -4,6 +4,7 @@ import ky from 'ky-universal'; import type { Alert, AlertApiResponse, + AlertEvaluationsApiResponse, AlertHistoryRangeApiResponse, AlertsApiResponse, InstallationApiResponse, @@ -22,7 +23,7 @@ import type { WebhookTestApiResponse, WebhookUpdateApiResponse, } from '@hyperdx/common-utils/dist/types'; -import { useMutation, useQuery } from '@tanstack/react-query'; +import { useInfiniteQuery, useMutation, useQuery } from '@tanstack/react-query'; import { IS_LOCAL_MODE } from './config'; import { getLocalDashboardTags } from './dashboard'; @@ -218,6 +219,39 @@ const api = { enabled: enabled && alertId != null, }); }, + getAlertEvaluationsQueryKey: ( + alertId: string | undefined, + startTime: number, + endTime: number, + ) => ['alertEvaluations', alertId, startTime, endTime] as const, + // Paginated evaluation history for the alert detail page: one entry per + // evaluation window (newest first), scoped to the given date range and + // including errors recorded for each window. Older pages are keyed off the + // server-provided `nextBefore` cursor, which advances even across gaps with + // no evaluations. Bounds are quantized to the minute so live ticks don't + // produce a new query key on every render. + useAlertEvaluations(alertId: string | undefined, dateRange: [Date, Date]) { + const BUCKET_MS = 60_000; + const startTime = + Math.floor(dateRange[0].getTime() / BUCKET_MS) * BUCKET_MS; + const endTime = Math.floor(dateRange[1].getTime() / BUCKET_MS) * BUCKET_MS; + return useInfiniteQuery({ + queryKey: api.getAlertEvaluationsQueryKey(alertId, startTime, endTime), + queryFn: ({ pageParam }) => + hdxServer(`alerts/${alertId}/evaluations`, { + method: 'GET', + searchParams: { + startTime, + endTime, + ...(pageParam != null && { before: pageParam }), + }, + }).json(), + initialPageParam: undefined as number | undefined, + getNextPageParam: lastPage => + lastPage.hasMore ? lastPage.nextBefore : undefined, + enabled: alertId != null && startTime < endTime, + }); + }, useServices() { return useQuery({ queryKey: [`services`], diff --git a/packages/app/src/components/AlertPreviewChart.tsx b/packages/app/src/components/AlertPreviewChart.tsx index 3165ef17af..03a96990f2 100644 --- a/packages/app/src/components/AlertPreviewChart.tsx +++ b/packages/app/src/components/AlertPreviewChart.tsx @@ -18,6 +18,7 @@ import { } from '@hyperdx/common-utils/dist/types'; import { Paper } from '@mantine/core'; +import { ChartAnnotation } from '@/components/charts/chartAnnotations'; import { DBTimeChart } from '@/components/DBTimeChart'; import { useAliasMapFromChartConfig } from '@/hooks/useChartConfig'; import { intervalToDateRange, intervalToGranularity } from '@/utils/alerts'; @@ -35,6 +36,11 @@ type AlertPreviewChartProps = { threshold: number; thresholdMax?: number; select?: string | null; + /** Override the interval-derived date range (e.g. alert detail page). */ + dateRange?: [Date, Date]; + /** Firing/recovery markers to draw on the chart. */ + annotations?: ChartAnnotation[]; + height?: number; }; export const AlertPreviewChart = ({ @@ -48,6 +54,9 @@ export const AlertPreviewChart = ({ thresholdMax, thresholdType, select, + dateRange, + annotations, + height = 200, }: AlertPreviewChartProps) => { const resolvedSelect = (select && select.trim().length > 0 @@ -86,11 +95,20 @@ export const AlertPreviewChart = ({ groupBy, select: ALERT_COUNT_DEFAULT_SELECT, displayType: DisplayType.Line, - dateRange: intervalToDateRange(interval), + dateRange: dateRange ?? intervalToDateRange(interval), granularity: intervalToGranularity(interval), }) as ChartConfigWithDateRange; return { ...chartConfig, with: aliasWith }; - }, [source, where, whereLanguage, filters, groupBy, interval, aliasWith]); + }, [ + source, + where, + whereLanguage, + filters, + groupBy, + interval, + aliasWith, + dateRange, + ]); const referenceLines = useMemo( () => @@ -103,13 +121,14 @@ export const AlertPreviewChart = ({ ); return ( - + diff --git a/packages/app/src/components/alerts/AlertDetailChart.tsx b/packages/app/src/components/alerts/AlertDetailChart.tsx new file mode 100644 index 0000000000..d9270ab33e --- /dev/null +++ b/packages/app/src/components/alerts/AlertDetailChart.tsx @@ -0,0 +1,275 @@ +import * as React from 'react'; +import Link from 'next/link'; +import { pick } from 'lodash'; +import { isTimeSeriesDisplayType } from '@hyperdx/common-utils/dist/core/utils'; +import { + isPromqlSavedChartConfig, + isRawSqlSavedChartConfig, +} from '@hyperdx/common-utils/dist/guards'; +import { + AlertSource, + ChartConfigWithDateRange, + DisplayType, + getSampleWeightExpression, + isLogSource, + isTraceSource, + SourceKind, +} from '@hyperdx/common-utils/dist/types'; +import { Anchor, Center, Paper, Skeleton, Text } from '@mantine/core'; + +import { AlertPreviewChart } from '@/components/AlertPreviewChart'; +import { getAlertReferenceLines } from '@/components/Alerts'; +import { DBTimeChart } from '@/components/DBTimeChart'; +import { useDashboards } from '@/dashboard'; +import { useAlertAnnotations } from '@/hooks/useAlertAnnotations'; +import { useSavedSearch } from '@/savedSearch'; +import { useSource } from '@/source'; +import type { AlertsPageItem } from '@/types'; +import { getMetricTableName } from '@/utils'; +import { intervalToGranularity } from '@/utils/alerts'; + +const CHART_HEIGHT = 280; + +function ChartShell({ children }: { children: React.ReactNode }) { + return ( + + {children} + + ); +} + +function ChartFallback({ + alertUrl, + message, +}: { + alertUrl?: string; + message: string; +}) { + return ( + +
+ + {message} + {alertUrl && ( + <> + {' '} + + Open source + + + )} + +
+ + ); +} + +function SavedSearchAlertChart({ + alert, + dateRange, +}: { + alert: AlertsPageItem; + dateRange: [Date, Date]; +}) { + const annotations = useAlertAnnotations(alert._id, dateRange, true); + const { data: savedSearch, isLoading: isSavedSearchLoading } = useSavedSearch( + { id: alert.savedSearchId ?? '' }, + { enabled: alert.savedSearchId != null }, + ); + const { data: source, isLoading: isSourceLoading } = useSource({ + id: savedSearch?.source, + }); + + if (isSavedSearchLoading || (savedSearch != null && isSourceLoading)) { + return ; + } + + if (!savedSearch || !source) { + return ( + + ); + } + + return ( + + ); +} + +function TileAlertChart({ + alert, + dateRange, + alertUrl, +}: { + alert: AlertsPageItem; + dateRange: [Date, Date]; + alertUrl?: string; +}) { + const annotations = useAlertAnnotations(alert._id, dateRange, true); + const { data: dashboards, isLoading: isDashboardsLoading } = useDashboards(); + const dashboard = dashboards?.find(d => d.id === alert.dashboardId); + const tile = dashboard?.tiles?.find(t => t.id === alert.tileId); + + const tileSourceId = + tile != null && !isPromqlSavedChartConfig(tile.config) + ? tile.config.source + : undefined; + const { data: source, isLoading: isSourceLoading } = useSource({ + id: tileSourceId, + }); + + const granularity = intervalToGranularity(alert.interval); + const config = React.useMemo(() => { + if (!tile || isPromqlSavedChartConfig(tile.config)) { + return undefined; + } + + // Raw SQL tiles: only time-series display types can be charted over the + // alert window (mirrors what the alert task evaluates as a time series). + if (isRawSqlSavedChartConfig(tile.config)) { + if (!isTimeSeriesDisplayType(tile.config.displayType)) { + return undefined; + } + if (!tile.config.source) { + return { ...tile.config, dateRange, granularity }; + } + if (!source) { + return undefined; + } + return { + ...tile.config, + ...pick(source, [ + 'implicitColumnExpression', + 'useTextIndexForImplicitColumn', + 'from', + 'metricTables', + ]), + ...(isLogSource(source) + ? { bodyExpression: source.bodyExpression } + : {}), + sampleWeightExpression: getSampleWeightExpression(source), + dateRange, + granularity, + }; + } + + // Builder tiles (mirrors the dashboard Tile's config assembly). Number + // tiles are rendered as a line chart here — the alert task evaluates them + // as a time series, and the threshold-over-time view is what matters. + if (!source?.connection) { + return undefined; + } + const isMetricSource = source.kind === SourceKind.Metric; + const firstSelect = tile.config.select[0]; + const metricType = + isMetricSource && typeof firstSelect !== 'string' + ? firstSelect?.metricType + : undefined; + const tableName = getMetricTableName(source, metricType); + return { + ...tile.config, + displayType: + tile.config.displayType === DisplayType.Number + ? DisplayType.Line + : tile.config.displayType, + connection: source.connection, + dateRange, + granularity, + timestampValueExpression: source.timestampValueExpression, + from: { + databaseName: source.from?.databaseName || 'default', + tableName: tableName || '', + }, + implicitColumnExpression: + isLogSource(source) || isTraceSource(source) + ? source.implicitColumnExpression + : undefined, + useTextIndexForImplicitColumn: + isLogSource(source) || isTraceSource(source) + ? source.useTextIndexForImplicitColumn + : undefined, + bodyExpression: isLogSource(source) ? source.bodyExpression : undefined, + sampleWeightExpression: getSampleWeightExpression(source), + metricTables: isMetricSource ? source.metricTables : undefined, + }; + }, [tile, source, dateRange, granularity]); + + const referenceLines = React.useMemo( + () => + getAlertReferenceLines({ + threshold: alert.threshold, + thresholdMax: alert.thresholdMax, + thresholdType: alert.thresholdType, + }), + [alert.threshold, alert.thresholdMax, alert.thresholdType], + ); + + if (isDashboardsLoading || (tileSourceId != null && isSourceLoading)) { + return ; + } + + if (!config) { + return ( + + ); + } + + return ( + + + + ); +} + +/** + * The alert's underlying query charted over the selected time range, with + * threshold reference lines and firing/recovery annotations. + */ +export function AlertDetailChart({ + alert, + dateRange, + alertUrl, +}: { + alert: AlertsPageItem; + dateRange: [Date, Date]; + alertUrl?: string; +}) { + if (alert.source === AlertSource.SAVED_SEARCH) { + return ; + } + if (alert.source === AlertSource.TILE) { + return ( + + ); + } + return ( + + ); +} diff --git a/packages/app/src/components/alerts/AlertEvaluationsTable.tsx b/packages/app/src/components/alerts/AlertEvaluationsTable.tsx new file mode 100644 index 0000000000..851fd2459e --- /dev/null +++ b/packages/app/src/components/alerts/AlertEvaluationsTable.tsx @@ -0,0 +1,396 @@ +import * as React from 'react'; +import { + ALERT_EVALUATION_GROUPS_LIMIT, + ALERT_INTERVAL_TO_MINUTES, + AlertEvaluation, + AlertEvaluationGroup, + AlertInterval, + AlertState, +} from '@hyperdx/common-utils/dist/types'; +import { + Badge, + Button, + Center, + Group, + Loader, + Skeleton, + Stack, + Table, + Text, + Tooltip, +} from '@mantine/core'; +import { useInViewport } from '@mantine/hooks'; +import { IconChevronDown, IconChevronRight } from '@tabler/icons-react'; + +import { + ALERT_ERROR_TYPE_LABELS, + AlertErrorsContent, +} from '@/components/alerts/AlertHistoryCards'; +import { FormatTime } from '@/useFormatTime'; +import { formatDurationMs } from '@/utils'; + +import styles from '@styles/AlertsPage.module.scss'; + +const TABLE_COLUMNS = 8; + +export function AlertStateBadge({ state }: { state: AlertState }) { + return stateBadge(state); +} + +function stateBadge(state: AlertState) { + switch (state) { + case AlertState.ALERT: + return ( + + Alert + + ); + case AlertState.PENDING: + return ( + + Pending + + ); + case AlertState.ERROR: + return ( + + Error + + ); + case AlertState.OK: + return Ok; + default: + return ( + + {state} + + ); + } +} + +function latestValue(history: AlertEvaluation): number | undefined { + const lastValues = history.lastValues; + if (!lastValues || lastValues.length === 0) { + return undefined; + } + return lastValues[lastValues.length - 1].count; +} + +/** Child row: one group's result within the parent evaluation window. */ +function GroupRow({ group }: { group: AlertEvaluationGroup }) { + return ( + + + + + └ + + + {group.group} + + + + {stateBadge(group.state)} + + {group.lastValue != null ? group.lastValue.count : '–'} + + {group.counts > 0 ? group.counts : '–'} + + + + + + ); +} + +/** Duration cell: formatted milliseconds, or a dash when not measured. */ +function durationCell(ms: number | undefined) { + return ms != null ? formatDurationMs(ms) : '–'; +} + +function EvaluationRow({ + history, + interval, +}: { + history: AlertEvaluation; + interval: AlertInterval; +}) { + const [expanded, setExpanded] = React.useState(false); + const errors = history.errors ?? []; + const groups = history.groups ?? []; + const hasErrors = errors.length > 0; + const hasGroups = groups.length > 0; + const value = hasGroups ? undefined : latestValue(history); + const errorTypes = Array.from(new Set(errors.map(e => e.type))); + + // The chart plots each bucket's value at the bucket *start*, while the + // evaluation runs at the bucket *end* (createdAt). Show the start of the + // latest evaluated bucket so the table lines up with the chart. + // `lastValues` are ascending, so the last entry is the newest bucket; + // failed evaluations have no lastValues, so fall back to + // createdAt − interval. + const lastValues = history.lastValues ?? []; + const lastBucketStart = + lastValues.length > 0 + ? lastValues[lastValues.length - 1].startTime + : new Date( + new Date(history.createdAt).getTime() - + ALERT_INTERVAL_TO_MINUTES[interval] * 60_000, + ); + const evaluatedSpanStart = lastValues[0]?.startTime ?? lastBucketStart; + + const groupsTotal = history.groupsTotal ?? groups.length; + const firingGroups = groups.filter(g => g.state === AlertState.ALERT).length; + const omittedGroups = groupsTotal - groups.length; + const backfilledBuckets = history.analytics?.backfilledBuckets ?? 0; + const expandable = hasErrors || hasGroups; + + return ( + <> + setExpanded(v => !v) : undefined} + style={expandable ? { cursor: 'pointer' } : undefined} + aria-expanded={expandable ? expanded : undefined} + > + + + {expandable ? ( + expanded ? ( + + ) : ( + + ) + ) : ( + // Keep timestamps aligned with expandable rows + + )} + + Evaluated –{' '} + + + } + withArrow + color="dark" + > + + + + + + + + + {stateBadge(history.state)} + {hasGroups && ( + + {firingGroups > 0 + ? `${firingGroups}/${groupsTotal} groups firing` + : `${groupsTotal} groups`} + + )} + + + {value != null ? value : '–'} + {history.counts > 0 ? history.counts : '–'} + + {backfilledBuckets > 0 ? ( + + + {backfilledBuckets} + + + ) : ( + '–' + )} + + {durationCell(history.analytics?.queryDurationMs)} + + {durationCell(history.analytics?.webhookDurationMs)} + + + {hasErrors ? ( + + {errorTypes.map(type => ALERT_ERROR_TYPE_LABELS[type]).join(', ')} + + ) : ( + '–' + )} + + + {expanded && ( + <> + {groups.map(group => ( + + ))} + {omittedGroups > 0 && ( + + + + Showing the top {ALERT_EVALUATION_GROUPS_LIMIT} of{' '} + {groupsTotal} groups (firing first) — additional groups + aren't fetched. + + + + )} + {hasErrors && ( + + + + + + + + )} + + )} + + ); +} + +/** + * Sentinel row at the bottom of the event stream: when scrolled into view + * (and older pages exist), it triggers the next fetch — infinite scroll in + * `limit`-sized chunks instead of a manual load-more button. + */ +function LoadMoreSentinel({ + isFetchingNextPage, + onLoadMore, +}: { + isFetchingNextPage: boolean; + onLoadMore: () => void; +}) { + const { ref, inViewport } = useInViewport(); + + React.useEffect(() => { + if (inViewport && !isFetchingNextPage) { + onLoadMore(); + } + }, [inViewport, isFetchingNextPage, onLoadMore]); + + return ( +
+ + + + Loading older evaluations… + + +
+ ); +} + +/** + * Datadog-style evaluation event stream: one parent row per evaluation + * window, newest first, expandable into per-group child rows for group-by + * alerts, with error details for failed evaluations. Fetches older windows + * in pages as the user scrolls to the bottom. + */ +export function AlertEvaluationsTable({ + evaluations, + interval, + isLoading, + isError, + hasNextPage, + isFetchingNextPage, + onLoadMore, +}: { + evaluations: AlertEvaluation[]; + interval: AlertInterval; + isLoading: boolean; + isError: boolean; + hasNextPage: boolean; + isFetchingNextPage: boolean; + onLoadMore: () => void; +}) { + if (isLoading) { + return ; + } + + if (evaluations.length === 0 && isError) { + return ( +
+ + Failed to load evaluations. + +
+ ); + } + + if (evaluations.length === 0 && !hasNextPage) { + return ( +
+ + No evaluations in the selected time range. + +
+ ); + } + + return ( + <> + + + + Evaluation Window + State + Latest Value + Breaches + Backfilled Buckets + Query Duration + Webhook Duration + Errors + + + + {evaluations.map(history => ( + + ))} + +
+ {hasNextPage && + // A failed page fetch must unmount the sentinel: its effect refires + // whenever isFetchingNextPage settles back to false, so leaving it + // mounted after an error would refetch in an unbounded loop. Show an + // explicit retry affordance instead. + (isError ? ( +
+ + + Failed to load older evaluations. + + + +
+ ) : ( + + ))} + + ); +} diff --git a/packages/app/src/components/alerts/AlertHistoryCards.tsx b/packages/app/src/components/alerts/AlertHistoryCards.tsx index a20537308f..1b3df3b37d 100644 --- a/packages/app/src/components/alerts/AlertHistoryCards.tsx +++ b/packages/app/src/components/alerts/AlertHistoryCards.tsx @@ -26,23 +26,83 @@ import styles from '@styles/AlertsPage.module.scss'; const HISTORY_ITEMS = 18; +export const ALERT_ERROR_TYPE_LABELS: Record = { + [AlertErrorType.INVALID_ALERT]: 'Invalid Configuration', + [AlertErrorType.QUERY_ERROR]: 'Query Error', + [AlertErrorType.QUERY_TIMEOUT]: 'Query Timeout', + [AlertErrorType.WEBHOOK_ERROR]: 'Webhook Error', + [AlertErrorType.UNKNOWN]: 'Unknown Error', +}; + function stateToBgColorClass(state: AlertState) { switch (state) { case AlertState.OK: return styles.ok; case AlertState.PENDING: return styles.pending; + case AlertState.ERROR: + return styles.error; default: return styles.alarm; } } +export function AlertErrorsContent({ errors }: { errors: AlertError[] }) { + return ( + + {errors.map((error, idx) => ( + + + {ALERT_ERROR_TYPE_LABELS[error.type]} at{' '} + + + + {error.message} + + + ))} + + ); +} + +/** Dedupe errors by type+message, keeping the most recent occurrence. */ +function dedupeAlertErrors(errors: AlertError[]): AlertError[] { + const map = new Map(); + for (const error of errors) { + const key = `${error.type}||${error.message}`; + const existing = map.get(key); + if ( + !existing || + new Date(error.timestamp).getTime() > + new Date(existing.timestamp).getTime() + ) { + map.set(key, error); + } + } + return Array.from(map.values()); +} + +function errorTypeSummary(errors: AlertError[]): string { + const types = Array.from(new Set(errors.map(error => error.type))); + return types.length === 1 + ? ALERT_ERROR_TYPE_LABELS[types[0]] + : 'Multiple Errors'; +} + function AlertHistoryCard({ history, alertUrl, + onShowErrors, }: { history: AlertHistory; alertUrl?: string; + onShowErrors: (history: AlertHistory) => void; }) { const start = new Date(history.createdAt.toString()); @@ -65,25 +125,41 @@ function AlertHistoryCard({ return url.pathname + url.search; }, [history, alertUrl]); + const isError = history.state === AlertState.ERROR; + const time = formatRelative(start, today); + const content = (
); - const count = history.counts ?? 0; - const pending = history.state === AlertState.PENDING ? 'pending' : ''; - const alert = `alert${count === 0 || count > 1 ? 's' : ''}`; - const time = formatRelative(start, today); - const label = `${count} ${pending} ${alert} ${time}`; + const label = React.useMemo(() => { + if (isError) { + const summary = errorTypeSummary(history.errors ?? []); + return `Evaluation failed (${summary}) ${time}. Click for details.`; + } + const count = history.counts ?? 0; + const pending = history.state === AlertState.PENDING ? 'pending' : ''; + const alert = `alert${count === 0 || count > 1 ? 's' : ''}`; + return `${count} ${pending} ${alert} ${time}`; + }, [isError, history, time]); return ( - {href ? ( + {isError ? ( + onShowErrors(history)} + aria-label="View evaluation errors" + > + {content} + + ) : href ? ( {content} @@ -94,40 +170,17 @@ function AlertHistoryCard({ ); } -const ALERT_ERROR_TYPE_LABELS: Record = { - [AlertErrorType.INVALID_ALERT]: 'Invalid Configuration', - [AlertErrorType.QUERY_ERROR]: 'Query Error', - [AlertErrorType.WEBHOOK_ERROR]: 'Webhook Error', - [AlertErrorType.UNKNOWN]: 'Unknown Error', -}; - function AlertErrorsIndicator({ alert }: { alert: AlertsPageItem }) { const [opened, { open, close }] = useDisclosure(false); - const { uniqueErrors, uniqueTypes } = React.useMemo(() => { - const map = new Map(); - for (const error of alert.executionErrors ?? []) { - const key = `${error.type}||${error.message}`; - const existing = map.get(key); - if ( - !existing || - new Date(error.timestamp).getTime() > - new Date(existing.timestamp).getTime() - ) { - map.set(key, error); - } - } - const errors = Array.from(map.values()); - const types = Array.from(new Set(errors.map(error => error.type))); - return { uniqueErrors: errors, uniqueTypes: types }; - }, [alert.executionErrors]); + const uniqueErrors = React.useMemo( + () => dedupeAlertErrors(alert.executionErrors ?? []), + [alert.executionErrors], + ); if (uniqueErrors.length === 0) return null; - const errorType = - uniqueTypes.length === 1 - ? ALERT_ERROR_TYPE_LABELS[uniqueTypes[0]] - : 'Multiple Errors'; + const errorType = errorTypeSummary(uniqueErrors); return ( <> @@ -159,25 +212,7 @@ function AlertErrorsIndicator({ alert }: { alert: AlertsPageItem }) { title="Alert Execution Errors" data-testid={`alert-error-modal-${alert._id}`} > - - {uniqueErrors.map((error, idx) => ( - - - {ALERT_ERROR_TYPE_LABELS[error.type]} at{' '} - - - - {error.message} - - - ))} - + ); @@ -186,28 +221,44 @@ function AlertErrorsIndicator({ alert }: { alert: AlertsPageItem }) { export function AlertHistoryCardList({ alert, alertUrl, + history: historyProp, + maxItems = HISTORY_ITEMS, + showErrorIndicator = true, }: { alert: AlertsPageItem; alertUrl?: string; + /** Evaluation windows to render; defaults to the alert's inline history. */ + history?: AlertHistory[]; + maxItems?: number; + showErrorIndicator?: boolean; }) { - const { history } = alert; + const history = historyProp ?? alert.history; + const [errorHistory, setErrorHistory] = React.useState( + null, + ); + const items = React.useMemo(() => { - if (history.length < HISTORY_ITEMS) { + if (history.length < maxItems) { return history; } - return history.slice(0, HISTORY_ITEMS); - }, [history]); + return history.slice(0, maxItems); + }, [history, maxItems]); const paddingItems = React.useMemo(() => { - if (history.length > HISTORY_ITEMS) { + if (history.length > maxItems) { return []; } - return new Array(HISTORY_ITEMS - history.length).fill(null); - }, [history]); + return new Array(maxItems - history.length).fill(null); + }, [history, maxItems]); + + const modalErrors = React.useMemo( + () => dedupeAlertErrors(errorHistory?.errors ?? []), + [errorHistory], + ); return ( - + {showErrorIndicator && } {items.length > 0 && (
{paddingItems.map((_, index) => ( @@ -223,10 +274,29 @@ export function AlertHistoryCardList({ key={index} history={history} alertUrl={alertUrl} + onShowErrors={setErrorHistory} /> ))}
)} + setErrorHistory(null)} + size="lg" + title={ + <> + Evaluation Errors + {errorHistory != null && ( + + + + )} + + } + data-testid={`alert-evaluation-error-modal-${alert._id}`} + > + + ); } diff --git a/packages/app/src/components/alerts/__tests__/AlertEvaluationsTable.test.tsx b/packages/app/src/components/alerts/__tests__/AlertEvaluationsTable.test.tsx new file mode 100644 index 0000000000..0be06f7fe3 --- /dev/null +++ b/packages/app/src/components/alerts/__tests__/AlertEvaluationsTable.test.tsx @@ -0,0 +1,292 @@ +import React from 'react'; +import { + AlertErrorType, + AlertEvaluation, + AlertState, +} from '@hyperdx/common-utils/dist/types'; +import { fireEvent, screen, within } from '@testing-library/react'; + +import { AlertEvaluationsTable } from '@/components/alerts/AlertEvaluationsTable'; + +// Controls whether the infinite-scroll sentinel reports itself as visible. +let mockInViewport = false; +jest.mock('@mantine/hooks', () => ({ + ...jest.requireActual('@mantine/hooks'), + useInViewport: () => ({ ref: jest.fn(), inViewport: mockInViewport }), +})); + +// Render raw ISO timestamps so time assertions don't depend on the test +// runner's timezone or the user's clock-format preference. +jest.mock('@/useFormatTime', () => ({ + FormatTime: ({ value }: { value?: number | string | Date }) => + value ? new Date(value).toISOString() : null, +})); + +const okWindow: AlertEvaluation = { + counts: 0, + createdAt: '2026-04-17T12:05:00.000Z', + lastValues: [{ startTime: '2026-04-17T12:00:00.000Z', count: 3 }], + state: AlertState.OK, +}; + +const errorWindow: AlertEvaluation = { + counts: 0, + createdAt: '2026-04-17T12:10:00.000Z', + lastValues: [], + state: AlertState.ERROR, + errors: [ + { + timestamp: '2026-04-17T12:11:00.000Z', + type: AlertErrorType.QUERY_TIMEOUT, + message: + 'Alert query did not complete within the 300s evaluation timeout.', + }, + ], +}; + +const groupedFiringWindow: AlertEvaluation = { + counts: 2, + createdAt: '2026-04-17T12:15:00.000Z', + lastValues: [ + { startTime: '2026-04-17T12:10:00.000Z', count: 1 }, + { startTime: '2026-04-17T12:10:00.000Z', count: 14 }, + ], + state: AlertState.ALERT, + groups: [ + { + group: 'ServiceName:api', + state: AlertState.ALERT, + counts: 2, + lastValue: { startTime: '2026-04-17T12:10:00.000Z', count: 14 }, + fired: true, + }, + { + group: 'ServiceName:web', + state: AlertState.OK, + counts: 0, + lastValue: { startTime: '2026-04-17T12:10:00.000Z', count: 1 }, + }, + ], + groupsTotal: 2, + analytics: { + queryDurationMs: 1200, + webhookDurationMs: 340, + backfilledBuckets: 0, + }, +}; + +const backfilledWindow: AlertEvaluation = { + counts: 0, + createdAt: '2026-04-17T12:20:00.000Z', + lastValues: [ + { startTime: '2026-04-17T12:05:00.000Z', count: 0 }, + { startTime: '2026-04-17T12:10:00.000Z', count: 0 }, + { startTime: '2026-04-17T12:15:00.000Z', count: 0 }, + ], + state: AlertState.OK, + analytics: { queryDurationMs: 800, backfilledBuckets: 2 }, +}; + +const renderTable = ( + props: Partial> = {}, +) => + renderWithMantine( + , + ); + +describe('AlertEvaluationsTable', () => { + beforeEach(() => { + mockInViewport = false; + }); + + it('renders one row per evaluation window with state badges', () => { + renderTable(); + + const rows = screen.getAllByTestId('alert-evaluation-row'); + expect(rows).toHaveLength(2); + expect(screen.getByText('Error')).toBeInTheDocument(); + expect(screen.getByText('Ok')).toBeInTheDocument(); + // OK row shows the latest value + expect(screen.getByText('3')).toBeInTheDocument(); + }); + + it('labels each row with the evaluated bucket start, matching the chart', () => { + renderTable({ + evaluations: [backfilledWindow, errorWindow, okWindow], + }); + + const rows = screen.getAllByTestId('alert-evaluation-row'); + // Backfilled window (createdAt 12:20, buckets 12:05/12:10/12:15) is + // labeled with the newest evaluated bucket, not the evaluation time + expect(rows[0]).toHaveTextContent('2026-04-17T12:15:00.000Z'); + expect(rows[0]).not.toHaveTextContent('2026-04-17T12:20:00.000Z'); + // Failed evaluation has no lastValues: falls back to createdAt − interval + expect(rows[1]).toHaveTextContent('2026-04-17T12:05:00.000Z'); + expect(rows[1]).not.toHaveTextContent('2026-04-17T12:10:00.000Z'); + // OK window (createdAt 12:05) shows its bucket start 12:00 + expect(rows[2]).toHaveTextContent('2026-04-17T12:00:00.000Z'); + expect(rows[2]).not.toHaveTextContent('2026-04-17T12:05:00.000Z'); + }); + + it('shows the error type label and expands to the full message', () => { + renderTable(); + + expect(screen.getByText('Query Timeout')).toBeInTheDocument(); + expect( + screen.queryByText(/did not complete within the 300s/), + ).not.toBeInTheDocument(); + + fireEvent.click(screen.getByText('Query Timeout')); + + expect( + screen.getByText(/did not complete within the 300s/), + ).toBeInTheDocument(); + }); + + it('fetches the next page when the scroll sentinel enters the viewport', () => { + mockInViewport = true; + const onLoadMore = jest.fn(); + renderTable({ hasNextPage: true, onLoadMore }); + + expect( + screen.getByTestId('alert-evaluations-load-more'), + ).toBeInTheDocument(); + expect(onLoadMore).toHaveBeenCalled(); + }); + + it('does not fetch while a page is already being fetched', () => { + mockInViewport = true; + const onLoadMore = jest.fn(); + renderTable({ hasNextPage: true, isFetchingNextPage: true, onLoadMore }); + + expect(onLoadMore).not.toHaveBeenCalled(); + }); + + it('does not render the sentinel when there are no older pages', () => { + renderTable({ hasNextPage: false }); + expect( + screen.queryByTestId('alert-evaluations-load-more'), + ).not.toBeInTheDocument(); + }); + + it('stops auto-fetching and offers a retry once a page fetch fails', () => { + // Without this, the sentinel effect refires each time the failed fetch + // settles (isFetchingNextPage true -> false), refetching forever. + mockInViewport = true; + const onLoadMore = jest.fn(); + renderTable({ hasNextPage: true, isError: true, onLoadMore }); + + // The sentinel is unmounted, so nothing auto-fetches + expect( + screen.queryByTestId('alert-evaluations-load-more'), + ).not.toBeInTheDocument(); + expect(onLoadMore).not.toHaveBeenCalled(); + + // An explicit retry affordance replaces it + expect( + screen.getByTestId('alert-evaluations-load-error'), + ).toBeInTheDocument(); + fireEvent.click(screen.getByRole('button', { name: 'Retry' })); + expect(onLoadMore).toHaveBeenCalledTimes(1); + }); + + it('shows a failure message when the first page fails to load', () => { + renderTable({ evaluations: [], isError: true }); + expect(screen.getByTestId('alert-evaluations-error')).toBeInTheDocument(); + expect( + screen.queryByText(/No evaluations in the selected time range/), + ).not.toBeInTheDocument(); + }); + + it('renders an empty state when the range has no evaluations', () => { + renderTable({ evaluations: [] }); + expect( + screen.getByText(/No evaluations in the selected time range/), + ).toBeInTheDocument(); + }); + + it('renders grouped windows collapsed, expanding into per-group child rows on click', () => { + renderTable({ + evaluations: [groupedFiringWindow, errorWindow, okWindow], + }); + + // Group summary on the parent row, but children stay collapsed — + // no rows auto-expand + expect(screen.getByText('1/2 groups firing')).toBeInTheDocument(); + expect(screen.queryAllByTestId('alert-evaluation-group-row')).toHaveLength( + 0, + ); + + fireEvent.click(screen.getByText('1/2 groups firing')); + + // Expanded: per-group child rows, firing first + const groupRows = screen.getAllByTestId('alert-evaluation-group-row'); + expect(groupRows).toHaveLength(2); + expect(groupRows[0]).toHaveTextContent('ServiceName:api'); + expect(groupRows[0]).toHaveTextContent('Alert'); + expect(groupRows[0]).toHaveTextContent('14'); + expect(groupRows[1]).toHaveTextContent('ServiceName:web'); + expect(groupRows[1]).toHaveTextContent('Ok'); + + // Clicking again collapses + fireEvent.click(screen.getByText('1/2 groups firing')); + expect(screen.queryAllByTestId('alert-evaluation-group-row')).toHaveLength( + 0, + ); + }); + + it('explains the server-side group cap when groups were omitted', () => { + renderTable({ + evaluations: [{ ...groupedFiringWindow, groupsTotal: 60 }], + }); + + expect(screen.getByText('1/60 groups firing')).toBeInTheDocument(); + + fireEvent.click(screen.getByText('1/60 groups firing')); + + expect( + screen.getByText( + /Showing the top 50 of 60 groups \(firing first\) — additional groups aren't fetched\./, + ), + ).toBeInTheDocument(); + }); + + it('hides the window-level value for grouped windows', () => { + renderTable({ evaluations: [groupedFiringWindow] }); + + const parentRow = screen.getByTestId('alert-evaluation-row'); + // Latest Value cell shows a dash; per-group values live in child rows + expect(parentRow).not.toHaveTextContent('14'); + }); + + it('shows backfilled buckets on the parent row when ticks were missed', () => { + renderTable({ evaluations: [backfilledWindow, okWindow] }); + + const rows = screen.getAllByTestId('alert-evaluation-row'); + // Backfilled Buckets is the 5th column + expect(within(rows[0]).getAllByRole('cell')[4]).toHaveTextContent('2'); + // Steady-state window shows a dash in the Backfilled Buckets column + expect(within(rows[1]).getAllByRole('cell')[4]).toHaveTextContent('–'); + }); + + it('shows query and webhook durations as columns on the parent row', () => { + renderTable({ evaluations: [groupedFiringWindow, errorWindow] }); + + const rows = screen.getAllByTestId('alert-evaluation-row'); + // Visible without expanding anything + expect(rows[0]).toHaveTextContent('1.2s'); + expect(rows[0]).toHaveTextContent('340ms'); + // Window without analytics shows dashes in the duration columns + expect(rows[1]).not.toHaveTextContent('1.2s'); + expect(rows[1]).not.toHaveTextContent('340ms'); + }); +}); diff --git a/packages/app/src/components/alerts/__tests__/AlertHistoryCards.test.tsx b/packages/app/src/components/alerts/__tests__/AlertHistoryCards.test.tsx new file mode 100644 index 0000000000..056ec7d5a3 --- /dev/null +++ b/packages/app/src/components/alerts/__tests__/AlertHistoryCards.test.tsx @@ -0,0 +1,71 @@ +import React from 'react'; +import { + AlertErrorType, + AlertHistory, + AlertState, +} from '@hyperdx/common-utils/dist/types'; +import { fireEvent, screen } from '@testing-library/react'; + +import { AlertHistoryCardList } from '@/components/alerts/AlertHistoryCards'; +import type { AlertsPageItem } from '@/types'; + +const makeAlert = (history: AlertHistory[]): AlertsPageItem => + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- partial test fixture + ({ + _id: 'alert-1', + interval: '5m', + threshold: 1, + thresholdType: 'above', + channel: { type: 'webhook' }, + createdAt: '2026-04-17T00:00:00.000Z', + updatedAt: '2026-04-17T00:00:00.000Z', + history, + }) as unknown as AlertsPageItem; + +const okWindow: AlertHistory = { + counts: 0, + createdAt: '2026-04-17T12:05:00.000Z', + lastValues: [{ startTime: '2026-04-17T12:00:00.000Z', count: 0 }], + state: AlertState.OK, +}; + +const errorWindow: AlertHistory = { + counts: 0, + createdAt: '2026-04-17T12:10:00.000Z', + lastValues: [], + state: AlertState.ERROR, + errors: [ + { + timestamp: '2026-04-17T12:11:00.000Z', + type: AlertErrorType.QUERY_ERROR, + message: 'clickhouse kaput', + }, + ], +}; + +describe('AlertHistoryCardList', () => { + it('renders errored evaluations as buttons that open the error details modal', async () => { + renderWithMantine( + , + ); + + const errorSegment = screen.getByRole('button', { + name: 'View evaluation errors', + }); + expect(errorSegment).toBeInTheDocument(); + expect(screen.queryByText('clickhouse kaput')).not.toBeInTheDocument(); + + fireEvent.click(errorSegment); + + // The modal content mounts asynchronously (Mantine portal + transition) + expect(await screen.findByText(/Query Error/)).toBeInTheDocument(); + expect(await screen.findByText('clickhouse kaput')).toBeInTheDocument(); + }); + + it('does not render error buttons for normal windows', () => { + renderWithMantine(); + expect( + screen.queryByRole('button', { name: 'View evaluation errors' }), + ).not.toBeInTheDocument(); + }); +}); diff --git a/packages/app/src/config.ts b/packages/app/src/config.ts index e9df70bf1e..2d8613892f 100644 --- a/packages/app/src/config.ts +++ b/packages/app/src/config.ts @@ -71,6 +71,10 @@ export const IS_METRICS_ENABLED = true; export const IS_MTVIEWS_ENABLED = false; export const IS_SESSIONS_ENABLED = true; export const IS_PROMQL_ENABLED = env('NEXT_PUBLIC_ENABLE_PROMQL') === 'true'; +// Alert detail page (/alerts/:id). Default off — currently enabled only in +// dev (.env.development) and CI (e2e webserver) while the feature bakes. +export const IS_ALERT_DETAILS_ENABLED = + env('NEXT_PUBLIC_ENABLE_ALERT_DETAILS') === 'true'; // Not exported: IS_IAC_EXPORT_ENABLED below is the only gate callers should // read. Leaving the raw flag importable re-opens the "forgot the local-mode // check" mistake that folding the two together was meant to close. diff --git a/packages/app/styles/AlertsPage.module.scss b/packages/app/styles/AlertsPage.module.scss index 6261985499..59fc0cf406 100644 --- a/packages/app/styles/AlertsPage.module.scss +++ b/packages/app/styles/AlertsPage.module.scss @@ -43,11 +43,29 @@ background-color: var(--color-bg-warning); } + // Errored evaluation: striped red so it reads as "bad" but is visually + // distinct from a solid firing (alarm) segment. + &.error { + background-color: var(--color-bg-danger); + background-image: repeating-linear-gradient( + -45deg, + transparent 0 2px, + rgb(0 0 0 / 45%) 2px 4px + ); + } + &.clickable { cursor: pointer; } } +// Child rows (per-group breakdown / error details) under an evaluation +// window row in the alert detail page's event stream. Muted background keeps +// parent windows visually dominant when several are expanded. +.evaluationChildRow { + background-color: var(--color-bg-muted); +} + .alertRow { display: flex; align-items: center; diff --git a/packages/app/tests/e2e/features/alerts.spec.ts b/packages/app/tests/e2e/features/alerts.spec.ts index c8bdb9e5be..464cab2827 100644 --- a/packages/app/tests/e2e/features/alerts.spec.ts +++ b/packages/app/tests/e2e/features/alerts.spec.ts @@ -718,6 +718,56 @@ test.describe( SEEDED_ERROR_ALERT.errorMessage, ); }); + + test('shows an errored evaluation in the history strip with details on click', async () => { + const seededCard = alertsPage.getAlertCardByName( + SEEDED_ERROR_ALERT.savedSearchName, + ); + await expect(seededCard).toBeVisible({ timeout: 10000 }); + + // The seeded ERROR evaluation window renders as a clickable segment + const errorSegment = alertsPage.getErrorHistorySegments(seededCard); + await expect(errorSegment).toHaveCount(1); + + await errorSegment.click(); + await expect(alertsPage.evaluationErrorModal).toBeVisible(); + await expect(alertsPage.evaluationErrorModal).toContainText( + 'Query Timeout', + ); + await expect( + alertsPage.evaluationErrorModal.locator('pre'), + ).toContainText(SEEDED_ERROR_ALERT.historyErrorMessage); + }); + + test('navigates to the alert detail page and shows the evaluation history', async () => { + const seededCard = alertsPage.getAlertCardByName( + SEEDED_ERROR_ALERT.savedSearchName, + ); + await expect(seededCard).toBeVisible({ timeout: 10000 }); + + await alertsPage.getDetailsLinkForAlertCard(seededCard).click(); + + // Generous timeout: in dev mode the first hit compiles the new route, + // which can take tens of seconds before the navigation completes. + await alertsPage.page.waitForURL(/\/alerts\/[a-f0-9]{24}/, { + timeout: 30000, + }); + await expect(alertsPage.detailPageContainer).toBeVisible({ + timeout: 15000, + }); + + // The event stream lists the errored evaluation with its type label + await expect(alertsPage.evaluationsTable).toBeVisible({ + timeout: 10000, + }); + await expect(alertsPage.evaluationsTable).toContainText('Query Timeout'); + // ...and the OK window seeded alongside it + await expect( + alertsPage.evaluationsTable.locator( + '[data-testid="alert-evaluation-row"]', + ), + ).toHaveCount(2); + }); }, ); diff --git a/packages/app/tests/e2e/global-setup-fullstack.ts b/packages/app/tests/e2e/global-setup-fullstack.ts index 866a3cd6f2..9c196b6900 100644 --- a/packages/app/tests/e2e/global-setup-fullstack.ts +++ b/packages/app/tests/e2e/global-setup-fullstack.ts @@ -56,6 +56,10 @@ export const SEEDED_ERROR_ALERT = { errorType: 'QUERY_ERROR', errorMessage: 'ClickHouse returned 500: DB::Exception: Timeout exceeded: elapsed 30s, maximum: 30s while executing query.', + // Message on the seeded ERROR-state AlertHistory row (evaluation history) + historyErrorType: 'QUERY_TIMEOUT', + historyErrorMessage: + 'Alert query did not complete within the 300s evaluation timeout. The evaluation is retried on every check, but the alert will not fire until the query completes in time.', }; /** @@ -379,6 +383,13 @@ async function seedAlertWithErrors( // check-alerts job is the only code that writes this field in normal // operation, so we write it here to avoid having to run that job during // setup. + // Evaluation windows aligned to the alert's 5m interval: one OK window + // followed by an ERROR window (a failed evaluation), so the alerts page + // history strip and the alert detail page have data to render. + const windowMs = 5 * 60 * 1000; + const errorWindowStart = Math.floor(Date.now() / windowMs) * windowMs; + const okWindowStart = errorWindowStart - windowMs; + const patchScript = ` use('hyperdx-e2e'); db.alerts.updateOne( @@ -396,6 +407,30 @@ db.alerts.updateOne( } } ); +db.alerthistories.deleteMany({ alert: ObjectId(${JSON.stringify(alertId)}) }); +db.alerthistories.insertMany([ + { + alert: ObjectId(${JSON.stringify(alertId)}), + createdAt: new Date(${okWindowStart}), + state: 'OK', + counts: 0, + lastValues: [{ startTime: new Date(${okWindowStart - windowMs}), count: 0 }] + }, + { + alert: ObjectId(${JSON.stringify(alertId)}), + createdAt: new Date(${errorWindowStart}), + state: 'ERROR', + counts: 0, + lastValues: [], + errors: [ + { + timestamp: new Date(), + type: ${JSON.stringify(SEEDED_ERROR_ALERT.historyErrorType)}, + message: ${JSON.stringify(SEEDED_ERROR_ALERT.historyErrorMessage)} + } + ] + } +]); `; try { diff --git a/packages/app/tests/e2e/page-objects/AlertsPage.ts b/packages/app/tests/e2e/page-objects/AlertsPage.ts index ab375c1e81..b82d4b76bf 100644 --- a/packages/app/tests/e2e/page-objects/AlertsPage.ts +++ b/packages/app/tests/e2e/page-objects/AlertsPage.ts @@ -115,6 +115,43 @@ export class AlertsPage { await icon.click(); } + /** + * Get the errored-evaluation segment(s) in an alert card's history strip. + * Rendered as clickable buttons (unlike normal segments which are links). + */ + getErrorHistorySegments(alertCard: Locator) { + return alertCard.getByRole('button', { name: 'View evaluation errors' }); + } + + /** + * The per-evaluation error details modal (opened by clicking an errored + * history segment). + */ + get evaluationErrorModal() { + return this.page.getByRole('dialog', { name: /Evaluation Errors/ }); + } + + /** + * Get the "Details" link for a given alert card (navigates to /alerts/:id). + */ + getDetailsLinkForAlertCard(alertCard: Locator) { + return alertCard.locator('[data-testid^="alert-details-link-"]'); + } + + /** + * The alert detail page root. + */ + get detailPageContainer() { + return this.page.locator('[data-testid="alert-detail-page"]'); + } + + /** + * The evaluation event-stream table on the alert detail page. + */ + get evaluationsTable() { + return this.page.locator('[data-testid="alert-evaluations-table"]'); + } + // --- Filter interactions --- get filters() { From bd81e590e351ec23dd23dc50a3b3e7bb718266c7 Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Thu, 6 Aug 2026 14:50:11 -0700 Subject: [PATCH 3/3] feat(alerts): persist evaluation errors and analytics in AlertHistory (HDX-4997) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When an alert evaluation fails (ClickHouse query error/timeout, webhook failure), the only persisted signal was alert.executionErrors — a latest-only snapshot wiped by the next successful run. - Failed evaluations are recorded as ERROR-state AlertHistory rows carrying error type/message/timestamp, upserted per evaluation window so per-tick retries collapse into a single row; rows expire with the existing 30d TTL. - Webhook/notification failures also produce an ERROR row alongside the normal evaluation rows; a stale ERROR row from a failed earlier tick is removed when a clean same-window retry succeeds. - Retry/backfill semantics are untouched: ERROR rows are excluded from the due-ness gate, the retry date-range computation, and consecutive-window counting — recording an error never marks the window as evaluated, so the failed window is still retried every tick and backfilled on recovery. - Query timeouts are classified as QUERY_TIMEOUT (client request timeout/abort, server-side TIMEOUT_EXCEEDED/159, socket timeouts — walking the cause chain since the query client wraps failures) with an actionable message that includes the configured evaluation timeout. - Evaluation analytics (queryDurationMs, webhookDurationMs, backfilledBuckets) are recorded on every history row the evaluation writes, including ERROR rows. --- .../alert-evaluation-error-persistence.md | 13 + packages/api/openapi.json | 1 + .../api/src/routers/external-api/v2/alerts.ts | 2 +- .../__tests__/checkAlerts.int.test.ts | 536 +++++++++++++++++- .../checkAlerts/__tests__/errors.test.ts | 134 +++++ packages/api/src/tasks/checkAlerts/errors.ts | 84 +++ packages/api/src/tasks/checkAlerts/index.ts | 111 +++- .../tasks/checkAlerts/providers/default.ts | 77 ++- .../src/tasks/checkAlerts/providers/index.ts | 25 +- packages/common-utils/src/clickhouse/index.ts | 9 + 10 files changed, 965 insertions(+), 27 deletions(-) create mode 100644 .changeset/alert-evaluation-error-persistence.md create mode 100644 packages/api/src/tasks/checkAlerts/__tests__/errors.test.ts diff --git a/.changeset/alert-evaluation-error-persistence.md b/.changeset/alert-evaluation-error-persistence.md new file mode 100644 index 0000000000..860110ffef --- /dev/null +++ b/.changeset/alert-evaluation-error-persistence.md @@ -0,0 +1,13 @@ +--- +'@hyperdx/common-utils': minor +'@hyperdx/api': minor +--- + +Persist alert evaluation errors (query errors, timeouts, webhook failures) as +ERROR-state AlertHistory records instead of only a latest-only snapshot, +upserted per evaluation window so retries collapse into a single row. Query +timeouts are classified separately (QUERY_TIMEOUT, including timeouts wrapped +by the ClickHouse query client) with an actionable message. ERROR rows are +excluded from scheduling/backfill computations so failed windows are still +retried and backfilled, and evaluation analytics (query/webhook durations, +backfilled buckets) are recorded on every history row. diff --git a/packages/api/openapi.json b/packages/api/openapi.json index c2fbbb9aee..ead070641f 100644 --- a/packages/api/openapi.json +++ b/packages/api/openapi.json @@ -118,6 +118,7 @@ "type": "string", "enum": [ "QUERY_ERROR", + "QUERY_TIMEOUT", "WEBHOOK_ERROR", "INVALID_ALERT", "UNKNOWN" diff --git a/packages/api/src/routers/external-api/v2/alerts.ts b/packages/api/src/routers/external-api/v2/alerts.ts index e53aedf4bc..4f0e53d24e 100644 --- a/packages/api/src/routers/external-api/v2/alerts.ts +++ b/packages/api/src/routers/external-api/v2/alerts.ts @@ -56,7 +56,7 @@ import { alertSchema, objectIdSchema } from '@/utils/zod'; * description: Channel type. * AlertErrorType: * type: string - * enum: [QUERY_ERROR, WEBHOOK_ERROR, INVALID_ALERT, UNKNOWN] + * enum: [QUERY_ERROR, QUERY_TIMEOUT, WEBHOOK_ERROR, INVALID_ALERT, UNKNOWN] * description: Category of error recorded during alert execution. * AlertExecutionError: * type: object diff --git a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts index 6700ef161e..86e4e76696 100644 --- a/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts +++ b/packages/api/src/tasks/checkAlerts/__tests__/checkAlerts.int.test.ts @@ -2849,12 +2849,26 @@ describe('checkAlerts', () => { teamWebhooksById, ); - // Alert should remain in its default OK state and no history/webhooks should be emitted + // Alert should remain in its default OK state and no normal + // history/webhooks should be emitted. The failure is recorded as an + // ERROR-state history row for the window. const updated = await Alert.findById(details.alert.id); expect(updated!.state).toBe('OK'); expect( - await AlertHistory.countDocuments({ alert: details.alert.id }), + await AlertHistory.countDocuments({ + alert: details.alert.id, + state: { $ne: AlertState.ERROR }, + }), ).toBe(0); + const errorHistories = await AlertHistory.find({ + alert: details.alert.id, + state: AlertState.ERROR, + }); + expect(errorHistories).toHaveLength(1); + expect(errorHistories[0].errors).toHaveLength(1); + expect(errorHistories[0].errors![0].type).toBe( + AlertErrorType.INVALID_ALERT, + ); expect(slack.postMessageToWebhook).not.toHaveBeenCalled(); // The invalid alert configuration should be recorded on the Alert @@ -2966,13 +2980,16 @@ describe('checkAlerts', () => { const updated = await Alert.findById(details.alert.id); // State must be untouched — still ALERT expect(updated!.state).toBe(AlertState.ALERT); - // No AlertHistory created + // No normal AlertHistory created expect( - await AlertHistory.countDocuments({ alert: details.alert.id }), + await AlertHistory.countDocuments({ + alert: details.alert.id, + state: { $ne: AlertState.ERROR }, + }), ).toBe(0); // No webhook fired expect(slack.postMessageToWebhook).not.toHaveBeenCalled(); - // Error recorded + // Error recorded on the alert (latest-only snapshot) expect(updated!.executionErrors).toBeDefined(); expect(updated!.executionErrors!.length).toBe(1); expect(updated!.executionErrors![0].type).toBe( @@ -2981,6 +2998,25 @@ describe('checkAlerts', () => { expect(updated!.executionErrors![0].message).toContain( 'clickhouse kaput', ); + // ...and persisted as an ERROR history row for the evaluation window + // (5m interval at 22:12 → window start 22:10) + const errorHistories = await AlertHistory.find({ + alert: details.alert.id, + state: AlertState.ERROR, + }); + expect(errorHistories).toHaveLength(1); + expect(errorHistories[0].createdAt.toISOString()).toBe( + '2023-11-16T22:10:00.000Z', + ); + expect(errorHistories[0].counts).toBe(0); + expect(errorHistories[0].lastValues).toHaveLength(0); + expect(errorHistories[0].errors).toHaveLength(1); + expect(errorHistories[0].errors![0].type).toBe( + AlertErrorType.QUERY_ERROR, + ); + expect(errorHistories[0].errors![0].message).toContain( + 'clickhouse kaput', + ); }); it('leaves OK state untouched when the ClickHouse query fails', async () => { @@ -3031,13 +3067,449 @@ describe('checkAlerts', () => { // Default state is OK — must stay OK (not flipped to ALERT or anything else) expect(updated!.state).toBe(AlertState.OK); expect( - await AlertHistory.countDocuments({ alert: details.alert.id }), + await AlertHistory.countDocuments({ + alert: details.alert.id, + state: { $ne: AlertState.ERROR }, + }), ).toBe(0); expect(updated!.executionErrors![0].type).toBe( AlertErrorType.QUERY_ERROR, ); }); + it('records a QUERY_TIMEOUT with an actionable message when the query times out', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + savedSearchId: savedSearch.id, + }, + { + taskType: AlertTaskType.SAVED_SEARCH, + savedSearch, + }, + ); + + // The exact message the ClickHouse client rejects with when + // request_timeout elapses. + jest + .spyOn(clickhouseClient, 'queryChartConfig') + .mockRejectedValueOnce(new Error('Timeout error.')); + + await processAlertAtTime( + new Date('2023-11-16T22:12:00.000Z'), + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + + const updated = await Alert.findById(details.alert.id); + expect(updated!.executionErrors).toHaveLength(1); + expect(updated!.executionErrors![0].type).toBe( + AlertErrorType.QUERY_TIMEOUT, + ); + expect(updated!.executionErrors![0].message).toMatch( + /did not complete within the \d+s evaluation timeout/, + ); + expect(updated!.executionErrors![0].message).toContain( + 'the alert will not fire until the query completes in time', + ); + + const errorHistories = await AlertHistory.find({ + alert: details.alert.id, + state: AlertState.ERROR, + }); + expect(errorHistories).toHaveLength(1); + expect(errorHistories[0].errors![0].type).toBe( + AlertErrorType.QUERY_TIMEOUT, + ); + // The ERROR row records the query's time-to-failure + expect(errorHistories[0].analytics?.queryDurationMs).toEqual( + expect.any(Number), + ); + }); + + it('dedupes error history rows per evaluation window and separates windows', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + savedSearchId: savedSearch.id, + }, + { + taskType: AlertTaskType.SAVED_SEARCH, + savedSearch, + }, + ); + + const querySpy = jest + .spyOn(clickhouseClient, 'queryChartConfig') + .mockRejectedValueOnce(new Error('boom 1')) + .mockRejectedValueOnce(new Error('boom 2')) + .mockRejectedValueOnce(new Error('boom 3')); + + // Two failing ticks within the same 5m window (22:10) — the retry + // must not be skipped (the ERROR row is excluded from the due-ness + // gate) and must not accumulate a second row (upsert per window). + await processAlertAtTime( + new Date('2023-11-16T22:12:00.000Z'), + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + await processAlertAtTime( + new Date('2023-11-16T22:13:00.000Z'), + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + + expect(querySpy).toHaveBeenCalledTimes(2); + let errorHistories = await AlertHistory.find({ + alert: details.alert.id, + state: AlertState.ERROR, + }); + expect(errorHistories).toHaveLength(1); + expect(errorHistories[0].createdAt.toISOString()).toBe( + '2023-11-16T22:10:00.000Z', + ); + // The row carries the latest attempt's error + expect(errorHistories[0].errors).toHaveLength(1); + expect(errorHistories[0].errors![0].message).toContain('boom 2'); + + // A failing tick in the NEXT window gets its own row + await processAlertAtTime( + new Date('2023-11-16T22:16:00.000Z'), + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + errorHistories = await AlertHistory.find({ + alert: details.alert.id, + state: AlertState.ERROR, + }).sort({ createdAt: 1 }); + expect(errorHistories).toHaveLength(2); + expect(errorHistories[1].createdAt.toISOString()).toBe( + '2023-11-16T22:15:00.000Z', + ); + + // No normal history was ever written + expect( + await AlertHistory.countDocuments({ + alert: details.alert.id, + state: { $ne: AlertState.ERROR }, + }), + ).toBe(0); + }); + + it('still evaluates and fires for a window whose earlier attempt failed', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + // Data inside the window evaluated at 22:10 (range [22:05, 22:10)) + await bulkInsertLogs([ + { + ServiceName: 'api', + Timestamp: new Date('2023-11-16T22:05:00.000Z'), + SeverityText: 'error', + Body: 'oh no', + }, + { + ServiceName: 'api', + Timestamp: new Date('2023-11-16T22:05:00.000Z'), + SeverityText: 'error', + Body: 'oh no', + }, + ]); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + savedSearchId: savedSearch.id, + }, + { + taskType: AlertTaskType.SAVED_SEARCH, + savedSearch, + }, + ); + + // First tick fails — records an ERROR row, no state/history change + jest + .spyOn(clickhouseClient, 'queryChartConfig') + .mockRejectedValueOnce(new Error('transient failure')); + + await processAlertAtTime( + new Date('2023-11-16T22:12:00.000Z'), + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + expect((await Alert.findById(details.alert.id))!.state).toBe( + AlertState.OK, + ); + + // Next tick (same window): the real query runs and the alert fires. + // If the ERROR row counted as "window evaluated", this tick would be + // skipped and the alert would never fire. + await processAlertAtTime( + new Date('2023-11-16T22:13:00.000Z'), + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + + const updated = await Alert.findById(details.alert.id); + expect(updated!.state).toBe(AlertState.ALERT); + // Errors on the alert are cleared by the successful execution + expect(updated!.executionErrors ?? []).toHaveLength(0); + + // The normal history was written for the retried window, and the + // failed attempt's ERROR row was removed — otherwise the window + // would render as ERROR forever (the evaluations view ranks ERROR + // above OK/ALERT) even though the retry succeeded. + const normalHistories = await AlertHistory.find({ + alert: details.alert.id, + state: { $ne: AlertState.ERROR }, + }); + expect(normalHistories).toHaveLength(1); + expect(normalHistories[0].state).toBe(AlertState.ALERT); + expect(normalHistories[0].createdAt.toISOString()).toBe( + '2023-11-16T22:10:00.000Z', + ); + expect( + await AlertHistory.countDocuments({ + alert: details.alert.id, + state: AlertState.ERROR, + }), + ).toBe(0); + // Evaluation analytics: single-window evaluation → no backfill; + // query duration recorded; notification sent → delivery time recorded + expect(normalHistories[0].analytics).toBeDefined(); + expect(normalHistories[0].analytics!.backfilledBuckets).toBe(0); + expect(normalHistories[0].analytics!.queryDurationMs).toEqual( + expect.any(Number), + ); + expect(normalHistories[0].analytics!.webhookDurationMs).toEqual( + expect.any(Number), + ); + }); + + it('keeps ERROR rows from older windows when a later window succeeds', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + // Non-breaching data in the 22:15 window (range [22:10, 22:15)) + await bulkInsertLogs([ + { + ServiceName: 'api', + Timestamp: new Date('2023-11-16T22:11:00.000Z'), + SeverityText: 'error', + Body: 'oh no', + }, + ]); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + savedSearchId: savedSearch.id, + }, + { + taskType: AlertTaskType.SAVED_SEARCH, + savedSearch, + }, + ); + + // Tick in the 22:10 window fails — ERROR row at 22:10 + jest + .spyOn(clickhouseClient, 'queryChartConfig') + .mockRejectedValueOnce(new Error('transient failure')); + await processAlertAtTime( + new Date('2023-11-16T22:12:00.000Z'), + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + + // The next window (22:15) evaluates cleanly. Only a same-window + // success clears an ERROR row — the 22:10 row is a truthful record + // of a tick that failed, so it must survive. + await processAlertAtTime( + new Date('2023-11-16T22:17:00.000Z'), + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + + const normalHistories = await AlertHistory.find({ + alert: details.alert.id, + state: { $ne: AlertState.ERROR }, + }); + expect(normalHistories).toHaveLength(1); + expect(normalHistories[0].createdAt.toISOString()).toBe( + '2023-11-16T22:15:00.000Z', + ); + + const errorHistories = await AlertHistory.find({ + alert: details.alert.id, + state: AlertState.ERROR, + }); + expect(errorHistories).toHaveLength(1); + expect(errorHistories[0].createdAt.toISOString()).toBe( + '2023-11-16T22:10:00.000Z', + ); + }); + + it('records backfilled buckets when an evaluation catches up missed windows', async () => { + const { + team, + webhook, + connection, + source, + savedSearch, + teamWebhooksById, + clickhouseClient, + } = await setupSavedSearchAlertTest(); + + const details = await createAlertDetails( + team, + source, + { + source: AlertSource.SAVED_SEARCH, + channel: { + type: 'webhook', + webhookId: webhook._id.toString(), + }, + interval: '5m', + thresholdType: AlertThresholdType.ABOVE, + threshold: 1, + savedSearchId: savedSearch.id, + }, + { + taskType: AlertTaskType.SAVED_SEARCH, + savedSearch, + }, + ); + + // First evaluation: steady state, covers a single bucket + await processAlertAtTime( + new Date('2023-11-16T22:12:00.000Z'), + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + const first = await AlertHistory.findOne({ + alert: details.alert.id, + createdAt: new Date('2023-11-16T22:10:00.000Z'), + }); + expect(first!.analytics!.backfilledBuckets).toBe(0); + + // Next evaluation runs 15 minutes late (missed the 22:15 and 22:20 + // ticks): the 22:25 window backfills the two missed buckets. + await processAlertAtTime( + new Date('2023-11-16T22:27:00.000Z'), + details, + clickhouseClient, + connection.id, + alertProvider, + teamWebhooksById, + ); + const second = await AlertHistory.findOne({ + alert: details.alert.id, + createdAt: new Date('2023-11-16T22:25:00.000Z'), + }); + expect(second).toBeDefined(); + expect(second!.analytics!.backfilledBuckets).toBe(2); + expect(second!.analytics!.queryDurationMs).toEqual(expect.any(Number)); + // No notification fired in this evaluation → no delivery time + expect(second!.analytics!.webhookDurationMs).toBeUndefined(); + }); + it.each([ { responseDescription: 'an error response', @@ -3141,10 +3613,31 @@ describe('checkAlerts', () => { const updated = await Alert.findById(details.alert.id); expect(updated!.state).toBe(AlertState.ALERT); - // Query succeeded, so AlertHistory should have been written + // Query succeeded, so normal AlertHistory should have been written expect( - await AlertHistory.countDocuments({ alert: details.alert.id }), + await AlertHistory.countDocuments({ + alert: details.alert.id, + state: { $ne: AlertState.ERROR }, + }), ).toBe(1); + // The webhook failure is also persisted as an ERROR history row + const errorHistories = await AlertHistory.find({ + alert: details.alert.id, + state: AlertState.ERROR, + }); + expect(errorHistories).toHaveLength(1); + expect(errorHistories[0].errors).toHaveLength(1); + expect(errorHistories[0].errors![0].type).toBe( + AlertErrorType.WEBHOOK_ERROR, + ); + expect(errorHistories[0].errors![0].message).toBe( + expectedErrorMessage, + ); + // ...carrying the evaluation's analytics, including the time spent + // attempting the webhook delivery + expect(errorHistories[0].analytics?.webhookDurationMs).toEqual( + expect.any(Number), + ); expect(updated!.executionErrors).toBeDefined(); expect(updated!.executionErrors!.length).toBe(1); expect(updated!.executionErrors![0].type).toBe( @@ -3525,10 +4018,25 @@ describe('checkAlerts', () => { expect(updated!.state).toBe(AlertState.ALERT); const histories = await AlertHistory.find({ alert: details.alert.id, + state: { $ne: AlertState.ERROR }, }); expect(histories.length).toBe(2); expect(histories.every(h => h.state === AlertState.ALERT)).toBe(true); + // Both groups' webhook failures land on a single ERROR history row + // for the evaluation window. + const errorHistories = await AlertHistory.find({ + alert: details.alert.id, + state: AlertState.ERROR, + }); + expect(errorHistories).toHaveLength(1); + expect(errorHistories[0].errors).toHaveLength(2); + expect( + errorHistories[0].errors!.every( + e => e.type === AlertErrorType.WEBHOOK_ERROR, + ), + ).toBe(true); + // Each group attempted to send a webhook and each one failed. withRetry // retries up to 3 times per group (2 groups × 3 attempts = 6 total calls). expect(fetchMock).toHaveBeenCalledTimes(6); @@ -3614,9 +4122,19 @@ describe('checkAlerts', () => { const updated = await Alert.findById(details.alert.id); // Query succeeded, state should flip to ALERT, history written + // (plus an ERROR row recording the webhook failure) expect(updated!.state).toBe(AlertState.ALERT); expect( - await AlertHistory.countDocuments({ alert: details.alert.id }), + await AlertHistory.countDocuments({ + alert: details.alert.id, + state: { $ne: AlertState.ERROR }, + }), + ).toBe(1); + expect( + await AlertHistory.countDocuments({ + alert: details.alert.id, + state: AlertState.ERROR, + }), ).toBe(1); // A WEBHOOK_ERROR should be recorded. The message is hardcoded for diff --git a/packages/api/src/tasks/checkAlerts/__tests__/errors.test.ts b/packages/api/src/tasks/checkAlerts/__tests__/errors.test.ts new file mode 100644 index 0000000000..8ba891fd4c --- /dev/null +++ b/packages/api/src/tasks/checkAlerts/__tests__/errors.test.ts @@ -0,0 +1,134 @@ +import { ClickHouseError } from '@clickhouse/client-common'; + +import { + isClientTimeoutOrAbortError, + isQueryTimeoutError, +} from '@/tasks/checkAlerts/errors'; + +describe('checkAlerts errors', () => { + describe('isQueryTimeoutError', () => { + it('detects the ClickHouse client request timeout', () => { + expect(isQueryTimeoutError(new Error('Timeout error.'))).toBe(true); + }); + + it('detects aborted requests', () => { + expect( + isQueryTimeoutError(new Error('The user aborted a request.')), + ).toBe(true); + }); + + it('detects server-side TIMEOUT_EXCEEDED by type', () => { + expect( + isQueryTimeoutError( + new ClickHouseError({ + code: '159', + type: 'TIMEOUT_EXCEEDED', + message: 'Timeout exceeded: elapsed 301.1 seconds', + }), + ), + ).toBe(true); + }); + + it('detects server-side timeouts by code when the type is not populated', () => { + expect( + isQueryTimeoutError( + new ClickHouseError({ + code: '159', + type: '', + message: 'Timeout exceeded', + }), + ), + ).toBe(true); + }); + + it('detects TCP-level socket timeouts', () => { + const err: NodeJS.ErrnoException = new Error('connect ETIMEDOUT'); + err.code = 'ETIMEDOUT'; + expect(isQueryTimeoutError(err)).toBe(true); + }); + + // BaseClickhouseClient.query wraps failures in a ClickHouseQueryError + // that copies the message but keeps the original error only on `cause`. + it('detects server-side TIMEOUT_EXCEEDED wrapped by the query client', () => { + const wrapper = new Error('Timeout exceeded: elapsed 301.1 seconds'); + wrapper.cause = new ClickHouseError({ + code: '159', + type: 'TIMEOUT_EXCEEDED', + message: 'Timeout exceeded: elapsed 301.1 seconds', + }); + expect(isQueryTimeoutError(wrapper)).toBe(true); + }); + + it('detects socket timeouts wrapped by the query client', () => { + const inner: NodeJS.ErrnoException = new Error('connect ETIMEDOUT'); + inner.code = 'ETIMEDOUT'; + const wrapper = new Error('connect ETIMEDOUT'); + wrapper.cause = inner; + expect(isQueryTimeoutError(wrapper)).toBe(true); + }); + + it('detects timeouts nested multiple causes deep', () => { + const inner = new ClickHouseError({ + code: '159', + type: 'TIMEOUT_EXCEEDED', + message: 'Timeout exceeded', + }); + const middle = new Error('query failed'); + middle.cause = inner; + const outer = new Error('evaluation failed'); + outer.cause = middle; + expect(isQueryTimeoutError(outer)).toBe(true); + }); + + it('does not classify wrapped non-timeout errors as timeouts', () => { + const wrapper = new Error("Table default.foo doesn't exist"); + wrapper.cause = new ClickHouseError({ + code: '60', + type: 'UNKNOWN_TABLE', + message: "Table default.foo doesn't exist", + }); + expect(isQueryTimeoutError(wrapper)).toBe(false); + }); + + it('terminates on self-referential cause chains', () => { + const err = new Error('recursive'); + err.cause = err; + expect(isQueryTimeoutError(err)).toBe(false); + }); + + it('does not classify other ClickHouse errors as timeouts', () => { + expect( + isQueryTimeoutError( + new ClickHouseError({ + code: '60', + type: 'UNKNOWN_TABLE', + message: "Table default.foo doesn't exist", + }), + ), + ).toBe(false); + expect(isQueryTimeoutError(new Error('clickhouse kaput'))).toBe(false); + expect(isQueryTimeoutError('Timeout error.')).toBe(false); + expect(isQueryTimeoutError(undefined)).toBe(false); + }); + }); + + describe('isClientTimeoutOrAbortError', () => { + it('only matches the client timeout/abort messages', () => { + expect(isClientTimeoutOrAbortError(new Error('Timeout error.'))).toBe( + true, + ); + expect( + isClientTimeoutOrAbortError(new Error('The user aborted a request.')), + ).toBe(true); + expect( + isClientTimeoutOrAbortError( + new ClickHouseError({ + code: '159', + type: 'TIMEOUT_EXCEEDED', + message: 'Timeout exceeded', + }), + ), + ).toBe(false); + }); + }); +}); diff --git a/packages/api/src/tasks/checkAlerts/errors.ts b/packages/api/src/tasks/checkAlerts/errors.ts index 37d7bb0b2c..73faa1994c 100644 --- a/packages/api/src/tasks/checkAlerts/errors.ts +++ b/packages/api/src/tasks/checkAlerts/errors.ts @@ -1,3 +1,5 @@ +import { ClickHouseError } from '@clickhouse/client-common'; + export const WEBHOOK_REDIRECT_ERROR_MESSAGE = 'Webhook destination responded with a redirect. Redirects are not supported.'; @@ -10,3 +12,85 @@ export class WebhookRedirectError extends Error { this.status = status; } } + +// @clickhouse/client (Node) rejects with these exact messages when the +// configured request_timeout elapses or the request is aborted. See +// clickhouse-js packages/client-node/src/connection/socket_pool.ts. +const CLIENT_TIMEOUT_MESSAGE = 'Timeout error.'; +const CLIENT_ABORT_MESSAGE = 'The user aborted a request.'; + +// ClickHouse server-side error for exceeded execution limits +// (e.g. max_execution_time): TIMEOUT_EXCEEDED, code 159. +const CH_TIMEOUT_TYPE = 'TIMEOUT_EXCEEDED'; +const CH_TIMEOUT_CODE = '159'; + +/** + * Check whether an error is a ClickHouseError, using both `instanceof` and a + * constructor-name fallback. The fallback handles the case where multiple + * copies of `@clickhouse/client-common` are installed (e.g. the api package + * uses one version while `common-utils` bundles another) — `instanceof` fails + * across the two class identities even though the shapes are identical. + */ +const isClickHouseError = ( + err: unknown, +): err is ClickHouseError & { type?: string; code?: string } => { + if (err instanceof ClickHouseError) return true; + return err instanceof Error && err.constructor?.name === 'ClickHouseError'; +}; + +/** + * Check whether an error is the ClickHouse client's own request_timeout + * firing (or the request being aborted) — i.e. the client-side evaluation + * timeout, as opposed to a server-side TIMEOUT_EXCEEDED. + */ +export const isClientTimeoutOrAbortError = (e: unknown): boolean => + e instanceof Error && + (e.message === CLIENT_TIMEOUT_MESSAGE || e.message === CLIENT_ABORT_MESSAGE); + +const isTimeoutErrorShallow = (e: Error): boolean => { + if (isClientTimeoutOrAbortError(e)) { + return true; + } + + if ( + isClickHouseError(e) && + (e.type === CH_TIMEOUT_TYPE || e.code === CH_TIMEOUT_CODE) + ) { + return true; + } + + return (e as NodeJS.ErrnoException).code === 'ETIMEDOUT'; +}; + +// Guard against pathological/self-referential cause chains. +const MAX_CAUSE_DEPTH = 5; + +/** + * Classify whether an alert query failure is a timeout/abort (rather than a + * query or connection error). Covers: + * - the ClickHouse client's request_timeout ("Timeout error.") + * - aborted requests ("The user aborted a request.") + * - server-side TIMEOUT_EXCEEDED (code 159, e.g. max_execution_time) + * - TCP-level socket timeouts (ETIMEDOUT) + * + * Walks the `cause` chain: `BaseClickhouseClient.query` wraps failures in a + * `ClickHouseQueryError` that preserves the message but keeps the original + * error (and its identifying type/code) only on `cause`, so server-side + * TIMEOUT_EXCEEDED and socket ETIMEDOUT would otherwise misclassify as + * generic query errors. + */ +export const isQueryTimeoutError = (e: unknown): boolean => { + const seen = new Set(); + let current = e; + for (let depth = 0; depth < MAX_CAUSE_DEPTH; depth++) { + if (!(current instanceof Error) || seen.has(current)) { + return false; + } + if (isTimeoutErrorShallow(current)) { + return true; + } + seen.add(current); + current = current.cause; + } + return false; +}; diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 81195eec23..981d04b6d0 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -51,12 +51,17 @@ import { serializeError } from 'serialize-error'; import { ALERT_HISTORY_QUERY_CONCURRENCY } from '@/controllers/alertHistory'; import { AlertState, IAlert, IAlertError } from '@/models/alert'; -import AlertHistory, { IAlertHistory } from '@/models/alertHistory'; +import AlertHistory, { + IAlertHistory, + IAlertHistoryAnalytics, +} from '@/models/alertHistory'; import { IDashboard } from '@/models/dashboard'; import { ISavedSearch } from '@/models/savedSearch'; import { ISource } from '@/models/source'; import { IWebhook } from '@/models/webhook'; import { + isClientTimeoutOrAbortError, + isQueryTimeoutError, WEBHOOK_REDIRECT_ERROR_MESSAGE, WebhookRedirectError, } from '@/tasks/checkAlerts/errors'; @@ -215,6 +220,29 @@ const getErrorMessage = (e: unknown): string => { return String(e); }; +const QUERY_TIMEOUT_RETRY_NOTE = + 'The evaluation is retried on every check, but the alert will not fire until the query completes in time.'; + +/** + * Build the IAlertError for a failed alert query, classifying timeouts + * (client request timeout/abort, server-side TIMEOUT_EXCEEDED, socket + * timeouts) separately from other query errors so the message is actionable. + */ +const makeQueryAlertError = ( + e: unknown, + requestTimeoutMs: number, +): IAlertError => { + if (!isQueryTimeoutError(e)) { + return makeAlertError(AlertErrorType.QUERY_ERROR, getErrorMessage(e)); + } + // For the client's own request timeout we know the configured limit; for + // server-side timeouts the original ClickHouse message carries the limit. + const message = isClientTimeoutOrAbortError(e) + ? `Alert query did not complete within the ${Math.round(requestTimeoutMs / 1000)}s evaluation timeout. ${QUERY_TIMEOUT_RETRY_NOTE}` + : `Alert query timed out before completing: ${getErrorMessage(e)}. ${QUERY_TIMEOUT_RETRY_NOTE}`; + return makeAlertError(AlertErrorType.QUERY_TIMEOUT, message); +}; + // Most webhook errors show a hardcoded message to avoid leaking sensitive request details in the UI. // Redirect errors are a known class of errors which we want to surface to the user, so it has a specific message. const makeWebhookAlertError = (error: unknown): IAlertError => { @@ -797,6 +825,13 @@ export const processAlert = async ( // availability SLIs cover every real evaluation regardless of exit point. const evalStartedAt = performance.now(); let evalOutcome: OperationOutcome | 'skipped' = 'success'; + // Scheduled start of the window being evaluated. Hoisted so the catch + // blocks can attribute error history records to the correct window. + let evaluationWindowStart: Date | undefined; + // Diagnostics persisted on every history record this evaluation writes + // (query duration, webhook delivery time, backfilled buckets). Populated + // progressively; hoisted so the catch blocks can attach what was measured. + const evaluationAnalytics: IAlertHistoryAnalytics = {}; try { const windowSizeInMins = ms(alert.interval) / 60000; const scheduleStartAt = normalizeScheduleStartAt({ @@ -838,6 +873,7 @@ export const processAlert = async ( scheduleOffsetMinutes, scheduleStartAt, ); + evaluationWindowStart = nowInMinsRoundDown; const hasGroupBy = alertHasGroupBy(details); // Check if we should skip this alert check based on last evaluation time @@ -971,31 +1007,53 @@ export const processAlert = async ( }); // SLO signal for alert data fetching (distinct from the end-to-end // evaluation SLI): did ClickHouse serve the alert query, and how fast. + const queryDurationMs = performance.now() - queryStartedAt; + evaluationAnalytics.queryDurationMs = Math.round(queryDurationMs); recordOperationOutcome({ operation: 'alerts.query', outcome: 'success', - durationMs: performance.now() - queryStartedAt, + durationMs: queryDurationMs, attributes: { alert_source: alert.source ?? 'unknown' }, }); } catch (e) { + const queryDurationMs = performance.now() - queryStartedAt; + // Time-to-failure — for QUERY_TIMEOUT this is roughly the configured + // evaluation timeout. + evaluationAnalytics.queryDurationMs = Math.round(queryDurationMs); recordOperationOutcome({ operation: 'alerts.query', outcome: 'error', - durationMs: performance.now() - queryStartedAt, + durationMs: queryDurationMs, attributes: { alert_source: alert.source ?? 'unknown' }, }); evalOutcome = 'error'; - alertQueryFailuresCounter.add(1); + const alertError = makeQueryAlertError( + e, + clickhouseClient.requestTimeoutMs, + ); + alertQueryFailuresCounter.add(1, { + error_type: + alertError.type === AlertErrorType.QUERY_TIMEOUT + ? 'timeout' + : 'error', + }); logger.error( { alertId: alert.id, + errorType: alertError.type, error: serializeError(e), }, 'Alert query failed, skipping state/history update', ); - await alertProvider.recordAlertErrors(alert.id, [ - makeAlertError(AlertErrorType.QUERY_ERROR, getErrorMessage(e)), - ]); + // Record the error on the alert and as an ERROR history row for this + // window. ERROR rows are excluded from the due-ness gate and date-range + // computation, so the failed window is still retried/backfilled. + await alertProvider.recordAlertErrors( + alert.id, + [alertError], + nowInMinsRoundDown, + evaluationAnalytics, + ); return; } @@ -1075,6 +1133,7 @@ export const processAlert = async ( : `Alert resolved for group "${group}", triggering ${alert.channel.type} notification`, ); + const notificationStartedAt = performance.now(); try { // Casts to any here because this is where I stopped unraveling the // alert logic requiring large, nested objects. We should look at @@ -1104,6 +1163,12 @@ export const processAlert = async ( 'Failed to fire channel event', ); executionErrors.push(makeWebhookAlertError(e)); + } finally { + // Total wall time spent delivering notifications in this evaluation + // (summed across groups/resolves, includes retries and failures). + evaluationAnalytics.webhookDurationMs = + (evaluationAnalytics.webhookDurationMs ?? 0) + + Math.round(performance.now() - notificationStartedAt); } }; @@ -1196,7 +1261,12 @@ export const processAlert = async ( // Auto-resolve await sendNotificationIfResolved(previous, history, ''); + // Single-value evaluations always cover exactly the current window. + evaluationAnalytics.backfilledBuckets = 0; const historyRecords = Array.from(histories.values()); + for (const record of historyRecords) { + record.analytics = evaluationAnalytics; + } await alertProvider.updateAlertState( alert.id, historyRecords, @@ -1211,6 +1281,12 @@ export const processAlert = async ( dateRange[1], `${windowSizeInMins} minute`, ); + // Buckets beyond the current window were backfilled in this run — + // earlier evaluation ticks were missed (job delay, failed evaluations). + evaluationAnalytics.backfilledBuckets = Math.max( + 0, + expectedBuckets.length - 1, + ); // Group data by time bucket (grouped alerts may have multiple entries per time bucket) const checkDataByBucket = new Map< @@ -1409,6 +1485,9 @@ export const processAlert = async ( // Save all history records and update alert state const historyRecords = Array.from(histories.values()); + for (const record of historyRecords) { + record.analytics = evaluationAnalytics; + } await alertProvider.updateAlertState( alert.id, historyRecords, @@ -1433,9 +1512,14 @@ export const processAlert = async ( ? AlertErrorType.INVALID_ALERT : AlertErrorType.UNKNOWN; try { - await alertProvider.recordAlertErrors(alert.id, [ - makeAlertError(type, message), - ]); + await alertProvider.recordAlertErrors( + alert.id, + [makeAlertError(type, message)], + evaluationWindowStart, + Object.keys(evaluationAnalytics).length > 0 + ? evaluationAnalytics + : undefined, + ); } catch (recordErr) { logger.error( { @@ -1507,6 +1591,10 @@ export const getPreviousAlertHistories = async ( $match: { alert: id, createdAt: { $lte: now, $gte: lookbackDate }, + // ERROR rows record failed evaluations; they must not count as + // "window evaluated" or the failed window would never be + // retried/backfilled. + state: { $ne: AlertState.ERROR }, }, }, // With a single alert value, the compound index {alert: 1, group: 1, createdAt: -1} @@ -1605,6 +1693,9 @@ export const getConsecutiveWindowHistories = async ( $match: { alert: id, createdAt: { $gte: earliestAllowedTime, $lt: windowStart }, + // Failed evaluations (ERROR rows) are not evaluated windows and + // must not affect consecutive-window counting. + state: { $ne: AlertState.ERROR }, }, }, { $sort: { alert: 1, group: 1, createdAt: -1 } }, diff --git a/packages/api/src/tasks/checkAlerts/providers/default.ts b/packages/api/src/tasks/checkAlerts/providers/default.ts index 079cdcc89d..30c89dea70 100644 --- a/packages/api/src/tasks/checkAlerts/providers/default.ts +++ b/packages/api/src/tasks/checkAlerts/providers/default.ts @@ -17,7 +17,10 @@ import Alert, { type IAlert, type IAlertError, } from '@/models/alert'; -import AlertHistory, { IAlertHistory } from '@/models/alertHistory'; +import AlertHistory, { + IAlertHistory, + IAlertHistoryAnalytics, +} from '@/models/alertHistory'; import Connection, { IConnection } from '@/models/connection'; import Dashboard from '@/models/dashboard'; import { type ISavedSearch, SavedSearch } from '@/models/savedSearch'; @@ -413,13 +416,83 @@ export default class DefaultAlertProvider implements AlertProvider { { _id: new mongoose.Types.ObjectId(alertId) }, { $set: { state: finalState, executionErrors: errors } }, ); + + // Notification (e.g. webhook) failures happened during this evaluation: + // record them as an ERROR history row alongside the normal rows so the + // failure is visible in the alert's evaluation history. All histories in + // one execution share the same createdAt (the evaluation window start) + // and the same evaluation-level analytics. + const evaluationWindowStart = histories[0]?.createdAt; + if (errors.length > 0 && evaluationWindowStart != null) { + await this.upsertErrorHistory( + alertId, + evaluationWindowStart, + errors, + histories[0]?.analytics, + ); + } else if ( + evaluationWindowStart != null && + successfulHistories.length > 0 + ) { + // A failed earlier tick may have left an ERROR row for this window + // (recordAlertErrors upserts one, and ERROR rows don't mark the window + // as evaluated so it's retried). The window has now evaluated cleanly — + // remove the stale ERROR row so the window doesn't render as ERROR + // forever (the evaluations view ranks ERROR above OK). + await AlertHistory.deleteOne({ + alert: new mongoose.Types.ObjectId(alertId), + createdAt: evaluationWindowStart, + state: AlertState.ERROR, + }); + } } - async recordAlertErrors(alertId: string, errors: IAlertError[]) { + async recordAlertErrors( + alertId: string, + errors: IAlertError[], + evaluationWindowStart?: Date, + analytics?: IAlertHistoryAnalytics, + ) { await Alert.updateOne( { _id: new mongoose.Types.ObjectId(alertId) }, { $set: { executionErrors: errors } }, ); + + if (evaluationWindowStart != null) { + await this.upsertErrorHistory( + alertId, + evaluationWindowStart, + errors, + analytics, + ); + } + } + + /** + * Upsert the ERROR-state history row for the given evaluation window. + * Keyed on {alert, createdAt, state} so retries within the same window + * update a single row instead of accumulating one row per tick — a + * permanently failing 1d alert produces one error row per day, not one per + * minute. Rows expire with the collection's existing TTL index. + */ + private async upsertErrorHistory( + alertId: string, + evaluationWindowStart: Date, + errors: IAlertError[], + analytics?: IAlertHistoryAnalytics, + ) { + await AlertHistory.updateOne( + { + alert: new mongoose.Types.ObjectId(alertId), + createdAt: evaluationWindowStart, + state: AlertState.ERROR, + }, + { + $set: { errors, ...(analytics != null && { analytics }) }, + $setOnInsert: { counts: 0, lastValues: [] }, + }, + { upsert: true }, + ); } async getWebhooks(teamId: string | ObjectId) { diff --git a/packages/api/src/tasks/checkAlerts/providers/index.ts b/packages/api/src/tasks/checkAlerts/providers/index.ts index aeed3e5407..159a352081 100644 --- a/packages/api/src/tasks/checkAlerts/providers/index.ts +++ b/packages/api/src/tasks/checkAlerts/providers/index.ts @@ -4,7 +4,7 @@ import _ from 'lodash'; import { ObjectId } from '@/models'; import { IAlert, IAlertError } from '@/models/alert'; -import { IAlertHistory } from '@/models/alertHistory'; +import { IAlertHistory, IAlertHistoryAnalytics } from '@/models/alertHistory'; import { IConnection } from '@/models/connection'; import { IDashboard } from '@/models/dashboard'; import { ISavedSearch } from '@/models/savedSearch'; @@ -83,6 +83,9 @@ export interface AlertProvider { * Uses Promise.allSettled to handle partial failures gracefully. * The alert state is determined from successfully saved histories, or falls back to all histories if all saves fail. * Also replaces the alert's `executionErrors` field with the provided errors from the current execution. + * When errors are present (e.g. webhook failures), an ERROR-state history + * row is additionally upserted for the evaluation window so the failure is + * visible in the alert's evaluation history. */ updateAlertState( alertId: string, @@ -91,11 +94,23 @@ export interface AlertProvider { ): Promise; /** - * Replace the alert's `executionErrors` field without changing state or creating history. - * Use this when an error prevents the normal state/history update from running - * (e.g. a ClickHouse query error). + * Replace the alert's `executionErrors` field without changing state or the + * alert's normal history. Use this when an error prevents the normal + * state/history update from running (e.g. a ClickHouse query error). + * + * When `evaluationWindowStart` is provided, the errors are also upserted as + * an ERROR-state AlertHistory row for that window (one row per window, + * regardless of retries). ERROR rows are excluded from scheduling/backfill + * computations, so recording them does not mark the window as evaluated. + * `analytics` carries whatever diagnostics the failed evaluation measured + * (e.g. the query's time-to-failure) onto that ERROR row. */ - recordAlertErrors(alertId: string, errors: IAlertError[]): Promise; + recordAlertErrors( + alertId: string, + errors: IAlertError[], + evaluationWindowStart?: Date, + analytics?: IAlertHistoryAnalytics, + ): Promise; /** Fetch all webhooks for the given team, returning a map of webhook ID to webhook */ getWebhooks(teamId: string | ObjectId): Promise>; diff --git a/packages/common-utils/src/clickhouse/index.ts b/packages/common-utils/src/clickhouse/index.ts index 7e24cd5793..4bdd8ba097 100644 --- a/packages/common-utils/src/clickhouse/index.ts +++ b/packages/common-utils/src/clickhouse/index.ts @@ -667,6 +667,15 @@ export abstract class BaseClickhouseClient { } } + /** + * The configured request timeout in milliseconds. Exposed so callers (e.g. + * the alert task) can produce actionable error messages when a query is + * aborted by this timeout. + */ + get requestTimeoutMs(): number { + return this.requestTimeout; + } + protected getClient(): WebClickHouseClient | NodeClickHouseClient { if (!this.client) { throw new Error(