;
+
+// Same stubbing strategy as DBTimeChartExemplarPin: the wiring under test lives
+// in DBTimeChart, so the chart and card are stubbed and driven through the
+// callbacks they're handed. `mock`-prefixed so jest's hoisted factories may close
+// over them.
+const mockChart = jest.fn((_props: ChartProps) => null);
+const mockCard = jest.fn((_props: CardProps) => null);
+const mockUseMe = jest.fn();
+const mockUseSource = jest.fn();
+const mockUseQueriedChartConfig = jest.fn();
+const mockUseExemplars = jest.fn();
+const mockUseExemplarTraceMeta = jest.fn();
+const mockRouterPush = jest.fn();
+
+jest.mock('@/HDXMultiSeriesTimeChart', () => ({
+ __esModule: true,
+ MemoChart: (props: ChartProps) => mockChart(props),
+}));
+
+jest.mock('@/components/Exemplars', () => ({
+ __esModule: true,
+ ExemplarHoverCard: (props: CardProps) => mockCard(props),
+}));
+
+jest.mock('@/hooks/useChartConfig', () => ({
+ useQueriedChartConfig: (...args: unknown[]) =>
+ mockUseQueriedChartConfig(...args),
+}));
+
+jest.mock('@/hooks/useMVOptimizationExplanation', () => ({
+ useMVOptimizationExplanation: jest
+ .fn()
+ .mockReturnValue({ data: undefined, isLoading: false }),
+}));
+
+jest.mock('@/api', () => ({
+ __esModule: true,
+ default: { useMe: (...args: unknown[]) => mockUseMe(...args) },
+}));
+
+jest.mock('@/source', () => ({
+ useSource: (...args: unknown[]) => mockUseSource(...args),
+ useChartNumberFormats: jest
+ .fn()
+ .mockReturnValue({ formatByColumn: new Map(), chartFormat: undefined }),
+}));
+
+jest.mock('@/hooks/useExemplars', () => ({
+ useExemplars: (...args: unknown[]) => mockUseExemplars(...args),
+ useExemplarTraceMeta: (...args: unknown[]) =>
+ mockUseExemplarTraceMeta(...args),
+}));
+
+jest.mock('next/router', () => ({
+ __esModule: true,
+ default: { push: (...args: unknown[]) => mockRouterPush(...args) },
+}));
+
+jest.mock('@/components/MaterializedViews/MVOptimizationIndicator', () =>
+ jest.fn(() => null),
+);
+jest.mock('@/components/charts/DateRangeIndicator', () => jest.fn(() => null));
+
+const exemplar: Exemplar = {
+ timestamp: 1704067200000,
+ value: 42,
+ traceId: 'abc123',
+};
+
+function lastProps(calls: [P][], what: string): P {
+ if (calls.length === 0) throw new Error(`${what} was never rendered`);
+ return calls[calls.length - 1][0];
+}
+const cardProps = () => lastProps(mockCard.mock.calls, 'ExemplarHoverCard');
+
+describe('DBTimeChart exemplar trace wiring', () => {
+ const baseConfig = {
+ dateRange: [new Date('2024-01-01'), new Date('2024-01-02')] as [Date, Date],
+ from: { databaseName: 'test', tableName: 'test' },
+ timestampValueExpression: 'timestamp',
+ connection: 'test-connection',
+ select: 'value',
+ where: '',
+ };
+
+ beforeEach(() => {
+ jest.clearAllMocks();
+ mockUseMe.mockReturnValue({
+ data: { team: { parallelizeWhenPossible: false } },
+ isLoading: false,
+ });
+ mockUseSource.mockReturnValue({ data: undefined, isLoading: false });
+ mockUseExemplars.mockReturnValue({
+ exemplars: [exemplar],
+ isLoading: false,
+ isError: false,
+ dropped: undefined,
+ });
+ mockUseExemplarTraceMeta.mockReturnValue({ data: null, isLoading: false });
+ mockUseQueriedChartConfig.mockReturnValue({
+ data: {
+ data: [{ timestamp: '2024-01-01 00:00:00', value: 100 }],
+ meta: [
+ { name: 'timestamp', type: 'DateTime' },
+ { name: 'value', type: 'Float64' },
+ ],
+ rows: 1,
+ isComplete: true,
+ },
+ isLoading: false,
+ isError: false,
+ isSuccess: true,
+ isPlaceholderData: false,
+ });
+ });
+
+ describe('trace source resolution', () => {
+ it("prefers the chart's explicit exemplarTraceSourceId over the source's linked one", () => {
+ mockUseSource.mockImplementation(({ id }: { id?: string }) =>
+ id === 'metric-source'
+ ? { data: { id: 'metric-source', traceSourceId: 'linked-traces' } }
+ : { data: { id, kind: 'trace' } },
+ );
+
+ renderWithMantine(
+
+
+
+ ,
+ );
+ }
+
if (toolbarSuffix && toolbarSuffix.length > 0) {
allToolbarItems.push(...toolbarSuffix);
}
return allToolbarItems;
}, [
+ exemplarNotice,
builderQueriedConfig,
config,
displayType,
diff --git a/packages/app/src/components/DBTimeChart/useExemplarCard.ts b/packages/app/src/components/DBTimeChart/useExemplarCard.ts
new file mode 100644
index 0000000000..5eaf006a4e
--- /dev/null
+++ b/packages/app/src/components/DBTimeChart/useExemplarCard.ts
@@ -0,0 +1,292 @@
+import { useCallback, useEffect, useRef, useState } from 'react';
+import Router from 'next/router';
+import {
+ ChartConfigWithDateRange,
+ DisplayType,
+ Exemplar,
+ SourceKind,
+ TSource,
+} from '@hyperdx/common-utils/dist/types';
+
+import { type PositionedExemplar } from '@/components/Exemplars';
+import { useExemplars, useExemplarTraceMeta } from '@/hooks/useExemplars';
+import { useSource } from '@/source';
+
+/**
+ * Half-width of the window the Inspect deep link opens around an exemplar. Wide
+ * enough to absorb clock skew between the metric pipeline and the trace store,
+ * narrow enough that the trace is not buried among unrelated ones.
+ */
+const EXEMPLAR_TRACE_WINDOW_MS = 5 * 60 * 1000;
+
+/** Epoch-ms [from, to] bracketing an exemplar, for the search page's range. */
+function exemplarTraceWindow(timestampMs: number): [number, number] {
+ return [
+ timestampMs - EXEMPLAR_TRACE_WINDOW_MS,
+ timestampMs + EXEMPLAR_TRACE_WINDOW_MS,
+ ];
+}
+
+/**
+ * Owns the exemplar overlay's data and its hover/pin card state for one chart.
+ *
+ * Extracted from DBTimeChart because this is a self-contained state machine —
+ * hover opens a card, a click pins it, a pin outranks hover, and several
+ * different events close it — and interleaving it with the chart's own tooltip
+ * state was what took that file past a thousand lines.
+ *
+ * The chart still coordinates: its drill-down tooltip and this card are mutually
+ * exclusive, so it calls `pin`/`unpin` alongside its own state updates rather
+ * than this hook reaching into the chart.
+ */
+export function useExemplarCard({
+ queriedConfig,
+ source,
+ displayType,
+ isPlotRendered,
+ plottedSeriesCount,
+}: {
+ queriedConfig: ChartConfigWithDateRange;
+ source: TSource | undefined;
+ /**
+ * Swapping this remounts the whole recharts subtree (Area vs Bar are different
+ * element types), so every ExemplarDot unmounts with no mouseleave and any open
+ * card would be left over markers that no longer exist.
+ */
+ displayType: DisplayType | undefined;
+ /**
+ * Whether the chart is currently drawing its plot. False for the loading,
+ * error and empty states, which replace the whole subtree — so the marker layer
+ * unmounts, no marker can report a position, and a card left open would come
+ * back anchored to coordinates from the previous chart instance.
+ */
+ isPlotRendered: boolean;
+ /** Series the chart actually draws; see useExemplars for why it matters. */
+ plottedSeriesCount?: number;
+}) {
+ // Exemplar overlay is configured per-chart via `enableExemplars` (set in the
+ // chart editor next to "As Ratio"), not a runtime toolbar toggle. The hook is
+ // a no-op unless the flag is set and the source kind supports exemplars.
+ const {
+ exemplars,
+ isError: isExemplarsError,
+ error: exemplarsError,
+ dropped: exemplarsDropped,
+ } = useExemplars(queriedConfig, source, plottedSeriesCount);
+
+ // A failed or suppressed exemplar scan otherwise looks exactly like "no
+ // exemplars in this range". Both are non-fatal — the chart itself is fine — so
+ // they surface as a toolbar indicator rather than replacing the chart. The
+ // upstream message is preferred over the generic fallback because the API
+ // phrases these actionably (e.g. "narrow the chart's time range").
+ const fetchNotice = isExemplarsError
+ ? (exemplarsError ??
+ 'Exemplars could not be loaded for this chart. The metric table may not carry Exemplars.* columns, or the Prometheus endpoint rejected the query.')
+ : exemplarsDropped === 'multiple-series'
+ ? 'Exemplars are hidden because this query returns more than one series. A marker sits at one trace’s own value, so it can’t be attributed across series yet — aggregate to a single line to see them.'
+ : null;
+
+ // Trace source an exemplar resolves against: the chart's explicit
+ // `exemplarTraceSourceId`, else the chart source's linked trace source.
+ const exemplarTraceSourceId =
+ queriedConfig.exemplarTraceSourceId ||
+ (source && 'traceSourceId' in source ? source.traceSourceId : undefined);
+ const { data: exemplarTraceSource } = useSource({
+ id: exemplarTraceSourceId,
+ });
+
+ // Hover card state. A short close delay lets the cursor travel from the SVG
+ // marker into the HTML card without it closing. Clicking a marker pins the
+ // same card open (`pinnedExemplar`), which then outranks hover until it's
+ // dismissed — via its close button, a click elsewhere on the chart, or
+ // another chart pinning something of its own.
+ const [hoveredExemplar, setHoveredExemplar] =
+ useState(null);
+ const [pinnedExemplar, setPinnedExemplar] =
+ useState(null);
+ const exemplarCloseTimerRef = useRef | null>(
+ null,
+ );
+ const openExemplarCard = useCallback(
+ (exemplar: Exemplar, x: number, y: number) => {
+ if (exemplarCloseTimerRef.current)
+ clearTimeout(exemplarCloseTimerRef.current);
+ setHoveredExemplar({ exemplar, x, y });
+ },
+ [],
+ );
+ const scheduleCloseExemplarCard = useCallback(() => {
+ if (exemplarCloseTimerRef.current)
+ clearTimeout(exemplarCloseTimerRef.current);
+ exemplarCloseTimerRef.current = setTimeout(
+ () => setHoveredExemplar(null),
+ 150,
+ );
+ }, []);
+ /**
+ * Re-anchor the open card to the marker's current position.
+ *
+ * The marker reports this itself (see ExemplarDot), which is the only place
+ * that knows: a zoom, a y-axis rescale and a container resize all move a marker
+ * without changing anything this hook can observe. Earlier attempts closed the
+ * card on proxies for movement instead — first the quantised date range, then
+ * the rendered x-domain — and each missed a cause. Following the marker also
+ * keeps a card the user deliberately pinned open through a live-tail tick.
+ *
+ * Only the marker the card is describing reports, so this is one call per
+ * change, not one per marker.
+ */
+ const moveExemplarCard = useCallback((x: number, y: number) => {
+ setPinnedExemplar(prev => (prev ? { ...prev, x, y } : prev));
+ setHoveredExemplar(prev => (prev ? { ...prev, x, y } : prev));
+ }, []);
+ useEffect(
+ () => () => {
+ if (exemplarCloseTimerRef.current)
+ clearTimeout(exemplarCloseTimerRef.current);
+ },
+ [],
+ );
+ // Note: when a refetch/re-thinning drops the hovered marker, the chart
+ // (MemoChart) detects the unmount against the actually-rendered points and
+ // fires onExemplarHoverEnd, which schedules this card's close — so no separate
+ // cleanup against the raw `exemplars` list is needed (that list is pre-thinning
+ // and would miss the re-thinning case anyway).
+
+ // A pin outranks hover, so the card doesn't swap contents under the cursor
+ // while the user is reading (or clicking) it.
+ const activeExemplar = pinnedExemplar ?? hoveredExemplar;
+
+ const unpinExemplarCard = useCallback(() => setPinnedExemplar(null), []);
+ // Key of the pinned marker, so the chart can close the card when a refetch or
+ // re-thinning drops that marker from the rendered set.
+ const pinnedExemplarKey = pinnedExemplar
+ ? `exemplar-${pinnedExemplar.exemplar.traceId}-${pinnedExemplar.exemplar.timestamp}`
+ : null;
+
+ // Closing both cards when the markers move lives in useExemplarMarkers, keyed
+ // on the rendered x-domain: that is what actually maps a data point to the
+ // pixels a card was positioned from, and this hook cannot see it. An earlier
+ // version keyed on the quantised date range here, which was both too coarse (a
+ // zoom inside the 30s bucket moved the markers and left the cards behind) and
+ // in the wrong place.
+
+ // Clear both cards whenever the marker layer goes away: a display-type switch
+ // remounts the recharts subtree (Area and Bar are different element types), and
+ // the loading, error and empty states replace it outright. In every case the
+ // markers a card was anchored to no longer exist.
+ useEffect(() => {
+ if (isPlotRendered) return;
+ setPinnedExemplar(null);
+ setHoveredExemplar(null);
+ }, [isPlotRendered]);
+
+ useEffect(() => {
+ setPinnedExemplar(null);
+ setHoveredExemplar(null);
+ }, [displayType]);
+
+ // Markers dropped by the render-layer clamps. Reported up from MemoChart because
+ // the drop happens after the fetch, so the fetch-layer `dropped` reason cannot
+ // see it and the overlay would otherwise thin out with no explanation.
+ const [clampDroppedCount, setClampDroppedCount] = useState(0);
+
+ // Escape closes the pinned card, matching the rest of the app's overlays.
+ useEffect(() => {
+ if (!pinnedExemplar) return;
+ const onKeyDown = (e: KeyboardEvent) => {
+ if (e.key === 'Escape') setPinnedExemplar(null);
+ };
+ document.addEventListener('keydown', onKeyDown);
+ return () => document.removeEventListener('keydown', onKeyDown);
+ }, [pinnedExemplar]);
+
+ const {
+ data: hoveredTraceMeta,
+ isLoading: isHoveredTraceMetaLoading,
+ isError: isHoveredTraceMetaError,
+ } = useExemplarTraceMeta(
+ activeExemplar?.exemplar.traceId,
+ exemplarTraceSource,
+ );
+
+ // A configured trace source that isn't actually a Trace kind never runs a query,
+ // so it looks identical to "no rows" — as does a failed query. Both used to read
+ // as "Trace not found in source", blaming the data for a misconfiguration or an
+ // error.
+ const traceLookupFailed =
+ isHoveredTraceMetaError ||
+ (!!exemplarTraceSource && exemplarTraceSource.kind !== SourceKind.Trace);
+
+ const navigateToExemplarTrace = useCallback(
+ (exemplar: Exemplar) => {
+ if (exemplarTraceSourceId) {
+ const params = new URLSearchParams();
+ params.set('source', exemplarTraceSourceId);
+ params.set('traceId', exemplar.traceId);
+ // Carry a window around the exemplar. Without from/to the search page
+ // falls back to the last 14 days (getDefaultDirectTraceDateRange), so a
+ // marker on a dashboard pinned to an older absolute range opened an empty
+ // trace view. The exemplar's own timestamp is the one thing we know for
+ // certain about where to look.
+ const [from, to] = exemplarTraceWindow(exemplar.timestamp);
+ params.set('from', String(from));
+ params.set('to', String(to));
+ Router.push(`/search?${params.toString()}`);
+ } else {
+ Router.push(`/trace/${encodeURIComponent(exemplar.traceId)}`);
+ }
+ },
+ [exemplarTraceSourceId],
+ );
+
+ /** Cancel a scheduled close — the cursor reached the card in time. */
+ const cancelClose = useCallback(() => {
+ if (exemplarCloseTimerRef.current) {
+ clearTimeout(exemplarCloseTimerRef.current);
+ }
+ }, []);
+
+ /**
+ * Pin the card for a clicked marker. Clears any pending hover-close and the
+ * hover card itself so the pinned contents can't be swapped out from under the
+ * cursor.
+ */
+ const pin = useCallback(
+ (exemplar: Exemplar, x: number, y: number) => {
+ cancelClose();
+ setHoveredExemplar(null);
+ setPinnedExemplar({ exemplar, x, y });
+ },
+ [cancelClose],
+ );
+
+ // The fetch-layer reason wins; a thinned overlay is the lesser problem and its
+ // note only appears when nothing worse is wrong.
+ const exemplarNotice =
+ fetchNotice ??
+ (clampDroppedCount > 0
+ ? `${clampDroppedCount} exemplar marker${clampDroppedCount === 1 ? '' : 's'} fall outside the chart's plotted range and are not drawn. A fitted y-axis floor (which a legend selection alone can produce) sits above them, or they belong to a different time window.`
+ : null);
+
+ return {
+ exemplars,
+ exemplarNotice,
+ reportClampDropped: setClampDroppedCount,
+ traceLookupFailed,
+ exemplarTraceSource,
+ exemplarTraceSourceId,
+ activeExemplar,
+ pinnedExemplar,
+ pinnedExemplarKey,
+ hoveredTraceMeta,
+ isHoveredTraceMetaLoading,
+ openExemplarCard,
+ scheduleCloseExemplarCard,
+ moveExemplarCard,
+ cancelClose,
+ pin,
+ unpin: unpinExemplarCard,
+ navigateToExemplarTrace,
+ };
+}
diff --git a/packages/app/src/components/Exemplars/ExemplarDot.stories.tsx b/packages/app/src/components/Exemplars/ExemplarDot.stories.tsx
new file mode 100644
index 0000000000..13db0628f1
--- /dev/null
+++ b/packages/app/src/components/Exemplars/ExemplarDot.stories.tsx
@@ -0,0 +1,69 @@
+import type { ReactNode } from 'react';
+import { Exemplar } from '@hyperdx/common-utils/dist/types';
+import type { Meta, StoryObj } from '@storybook/nextjs';
+
+import { ExemplarDot } from './ExemplarDot';
+
+/**
+ * Diamond marker overlaid on a time chart to mark an individual exemplar trace.
+ * It is rendered by recharts as a `` to show the
+ * marker in isolation. The fill uses the `--color-chart-warning` token — use the
+ * Theme (Light / Dark) and Brand toolbar toggles to review both.
+ */
+const meta = {
+ title: 'Components/Exemplars/ExemplarDot',
+ component: ExemplarDot,
+ parameters: { layout: 'centered' },
+} satisfies Meta;
+
+export default meta;
+type Story = StoryObj;
+
+const mockExemplar: Exemplar = {
+ timestamp: 1_700_000_000_000,
+ value: 128.4,
+ traceId: '4bf92f3577b34da6a3ce929d0e0e4736',
+ spanId: '00f067aa0ba902b7',
+};
+
+// A small chart-like canvas so the marker has a baseline for context.
+function SvgCanvas({ children }: { children: ReactNode }) {
+ return (
+
+
+ );
+}
+
+export const Default: Story = {
+ args: { cx: 140, cy: 80, exemplar: mockExemplar },
+ render: args => (
+
+
+ ),
+};
+
+export const AlongASeries: Story = {
+ args: { exemplar: mockExemplar },
+ render: args => (
+
+ {[40, 90, 140, 190, 240].map((cx, i) => (
+
+ ),
+};
diff --git a/packages/app/src/components/Exemplars/ExemplarDot.tsx b/packages/app/src/components/Exemplars/ExemplarDot.tsx
new file mode 100644
index 0000000000..4cded4e1ce
--- /dev/null
+++ b/packages/app/src/components/Exemplars/ExemplarDot.tsx
@@ -0,0 +1,84 @@
+import { useEffect } from 'react';
+import { Exemplar } from '@hyperdx/common-utils/dist/types';
+
+// Half-diagonal of the diamond marker, in px.
+const DIAMOND_HALF_SIZE = 4;
+// Radius of the transparent hit target that eases hovering the small marker.
+const HIT_RADIUS = 9;
+
+type ExemplarDotProps = {
+ // cx/cy are injected by recharts when this is used as a onHoverStart?.(exemplar, cx, cy)}
+ onMouseLeave={() => onHoverEnd?.()}
+ // The chart's own onClick pins a drill-down tooltip over the whole plot
+ // area; without stopping here, clicking a marker would open that instead
+ // of the exemplar's menu.
+ onClick={e => {
+ e.stopPropagation();
+ onSelect?.(exemplar, cx, cy);
+ }}
+ >
+
+ );
+}
diff --git a/packages/app/src/components/Exemplars/ExemplarHoverCard.stories.tsx b/packages/app/src/components/Exemplars/ExemplarHoverCard.stories.tsx
new file mode 100644
index 0000000000..9aca74fcd1
--- /dev/null
+++ b/packages/app/src/components/Exemplars/ExemplarHoverCard.stories.tsx
@@ -0,0 +1,103 @@
+import type { ReactNode } from 'react';
+import { Exemplar } from '@hyperdx/common-utils/dist/types';
+import type { Meta, StoryObj } from '@storybook/nextjs';
+
+import { ExemplarHoverCard } from './ExemplarHoverCard';
+
+/**
+ * Floating card shown when hovering an exemplar marker on a time chart. It shows
+ * the linked trace's metadata (resolved from the configured exemplar trace
+ * source) plus an "Inspect trace" button. The card is `position: absolute` and
+ * flips / clamps against its offset parent so it never overflows the chart, so
+ * every story wraps it in a relative, chart-sized container. Use the Theme
+ * (Light / Dark) and Brand toolbar toggles to review each state.
+ */
+const meta = {
+ title: 'Components/Exemplars/ExemplarHoverCard',
+ component: ExemplarHoverCard,
+ parameters: { layout: 'centered' },
+} satisfies Meta;
+
+export default meta;
+type Story = StoryObj;
+
+const mockExemplar: Exemplar = {
+ timestamp: 1_700_000_000_000,
+ value: 128.4,
+ traceId: '4bf92f3577b34da6a3ce929d0e0e4736',
+ spanId: '00f067aa0ba902b7',
+};
+
+const hovered = { exemplar: mockExemplar, x: 60, y: 40 };
+
+const noop = () => undefined;
+
+// A relative, chart-sized container so the card's absolute positioning and
+// flip/clamp logic resolve against a realistic offset parent.
+function ChartArea({ children }: { children: ReactNode }) {
+ return (
+
+ {children}
+
+ );
+}
+
+const baseArgs = {
+ hovered,
+ isLoading: false,
+ traceSourceConfigured: true,
+ onInspect: noop,
+ onMouseEnter: noop,
+ onMouseLeave: noop,
+};
+
+const render: Story['render'] = args => (
+
+
+);
+
+export const FullMetadata: Story = {
+ render,
+ args: {
+ ...baseArgs,
+ meta: {
+ service: 'checkout-api',
+ spanName: 'POST /checkout',
+ durationMs: 128.42,
+ statusCode: 'OK',
+ },
+ },
+};
+
+export const PartialMetadata: Story = {
+ render,
+ args: {
+ ...baseArgs,
+ meta: { service: 'checkout-api', durationMs: 128.42 },
+ },
+};
+
+export const Loading: Story = {
+ render,
+ args: { ...baseArgs, isLoading: true },
+};
+
+export const TraceNotFound: Story = {
+ render,
+ args: { ...baseArgs, meta: undefined },
+};
+
+export const NoTraceSourceConfigured: Story = {
+ render,
+ args: { ...baseArgs, traceSourceConfigured: false },
+};
diff --git a/packages/app/src/components/Exemplars/ExemplarHoverCard.tsx b/packages/app/src/components/Exemplars/ExemplarHoverCard.tsx
new file mode 100644
index 0000000000..fe6fb0c4d3
--- /dev/null
+++ b/packages/app/src/components/Exemplars/ExemplarHoverCard.tsx
@@ -0,0 +1,175 @@
+import { useLayoutEffect, useRef, useState } from 'react';
+import { Exemplar, NumberFormat } from '@hyperdx/common-utils/dist/types';
+import { Button, CloseButton, Group, Paper, Stack, Text } from '@mantine/core';
+
+import type { PositionedExemplar } from '@/components/Exemplars/exemplarPoints';
+import type { ExemplarTraceMeta } from '@/hooks/useExemplars';
+import { useFormatTime } from '@/useFormatTime';
+import { formatNumber } from '@/utils';
+
+type ExemplarHoverCardProps = {
+ /** The hovered exemplar plus its on-screen position; null hides the card. */
+ hovered: PositionedExemplar | null;
+ /** Trace metadata resolved from the configured exemplar trace source. */
+ meta?: ExemplarTraceMeta;
+ isLoading: boolean;
+ /** Whether an exemplar trace source is configured for this chart. */
+ traceSourceConfigured: boolean;
+ /**
+ * The trace lookup could not run or failed — a misconfigured source kind or a
+ * query error. Distinct from "no rows": both used to read as "not found",
+ * blaming the data for a problem that is not the data's.
+ */
+ traceLookupFailed?: boolean;
+ /** Chart's number format, so the exemplar's value reads like the y-axis. */
+ numberFormat?: NumberFormat;
+ /**
+ * Clicking a marker pins the card open: it stops following the cursor and
+ * only closes via `onClose` (or a click elsewhere on the chart).
+ */
+ pinned?: boolean;
+ onClose?: () => void;
+ onInspect: (exemplar: Exemplar) => void;
+ onMouseEnter: () => void;
+ onMouseLeave: () => void;
+};
+
+/**
+ * Floating card shown when hovering an exemplar marker: trace metadata (from the
+ * configured exemplar trace source) plus a button to open the trace directly.
+ */
+export function ExemplarHoverCard({
+ hovered,
+ meta,
+ isLoading,
+ traceSourceConfigured,
+ traceLookupFailed = false,
+ numberFormat,
+ pinned = false,
+ onClose,
+ onInspect,
+ onMouseEnter,
+ onMouseLeave,
+}: ExemplarHoverCardProps) {
+ const formatTime = useFormatTime();
+ const ref = useRef(null);
+ const [pos, setPos] = useState<{ left: number; top: number } | null>(null);
+
+ // Position the card next to the marker, but flip to the left / clamp upward
+ // when it would overflow the chart container, so it's never cut off. Measured
+ // after render (size depends on the async-loaded metadata).
+ useLayoutEffect(() => {
+ if (!hovered || !ref.current) {
+ setPos(null);
+ return;
+ }
+ const el = ref.current;
+ const parent = el.offsetParent;
+ const pW = parent?.clientWidth ?? window.innerWidth;
+ const pH = parent?.clientHeight ?? window.innerHeight;
+ const cardW = el.offsetWidth;
+ const cardH = el.offsetHeight;
+ const margin = 12;
+
+ let left = hovered.x + margin;
+ if (left + cardW > pW) left = hovered.x - margin - cardW; // flip left
+ left = Math.max(4, Math.min(left, pW - cardW - 4));
+
+ let top = hovered.y - margin;
+ if (top + cardH > pH) top = pH - cardH - 4; // shift up to stay in view
+ top = Math.max(4, top);
+
+ setPos({ left, top });
+ }, [hovered, meta, isLoading, traceSourceConfigured]);
+
+ if (!hovered) return null;
+ const { exemplar } = hovered;
+ return (
+ e.stopPropagation()}
+ >
+
+
+
+
+ Exemplar
+
+
+
+ {exemplar.traceId.slice(0, 16)}…
+
+ {pinned && (
+
+
+ {/*
+ The exemplar's own value and time, always shown. The marker's drawn
+ position is not trustworthy on its own: clampExemplarY pins it into
+ the y-domain and clampExemplarX into the x-domain, so a marker can sit
+ up to a bucket away in time and at the axis edge in value. These two
+ rows are what make that trade-off honest, which is why they render
+ here rather than inside the trace-source branch below — a chart with
+ no trace source configured still needs them.
+ */}
+
+
+ Value: {formatNumber(exemplar.value, numberFormat)}
+
+ Time: {formatTime(exemplar.timestamp)}
+
+ {!traceSourceConfigured ? (
+
+ Set an exemplar trace source in the chart editor to see trace
+ details.
+
+ ) : isLoading ? (
+
+ Loading trace…
+
+ ) : traceLookupFailed ? (
+
+ Trace details could not be loaded. Check the exemplar trace source
+ is a trace source and the query succeeded.
+
+ ) : meta ? (
+
+ {meta.service && Service: {meta.service} }
+ {meta.spanName && Span: {meta.spanName} }
+ {meta.durationMs != null && (
+ Duration: {meta.durationMs.toFixed(1)} ms
+ )}
+ {meta.statusCode && (
+ Status: {meta.statusCode}
+ )}
+
+ ) : (
+
+ Trace not found in source.
+
+ )}
+ onInspect(exemplar)}
+ >
+ Inspect trace
+
+
+
+
>) {
+ const onParentClick = jest.fn();
+ const { container } = render(
+
+ ,
+ );
+ // The transparent hit circle is the marker's click target.
+ const hitTarget = container.querySelector('circle')!;
+ return { onParentClick, hitTarget };
+}
+
+describe('ExemplarDot', () => {
+ it('reports a click and keeps it from reaching the chart', () => {
+ const onSelect = jest.fn();
+ const { onParentClick, hitTarget } = renderDot({ onSelect });
+
+ fireEvent.click(hitTarget);
+
+ // Without the stopPropagation this fix adds, the chart's own onClick also
+ // fires and opens the drill-down tooltip over the exemplar's menu.
+ expect(onSelect).toHaveBeenCalledWith(exemplar, 10, 20);
+ expect(onParentClick).not.toHaveBeenCalled();
+ });
+
+ it('swallows the click even with no handler attached', () => {
+ const { onParentClick, hitTarget } = renderDot({});
+
+ fireEvent.click(hitTarget);
+
+ expect(onParentClick).not.toHaveBeenCalled();
+ });
+
+ it('reports hover enter/leave with the marker position', () => {
+ const onHoverStart = jest.fn();
+ const onHoverEnd = jest.fn();
+ const { hitTarget } = renderDot({ onHoverStart, onHoverEnd });
+
+ fireEvent.mouseEnter(hitTarget.parentElement!);
+ fireEvent.mouseLeave(hitTarget.parentElement!);
+
+ expect(onHoverStart).toHaveBeenCalledWith(exemplar, 10, 20);
+ expect(onHoverEnd).toHaveBeenCalled();
+ });
+
+ it('renders nothing until recharts supplies coordinates', () => {
+ const { container } = render(
+
+ ,
+ );
+ expect(container.querySelector('path')).toBeNull();
+ });
+});
+
+/**
+ * Recharts recomputes cx/cy whenever the scales or the container change, so this
+ * marker is the only thing that knows an open card's anchor has gone stale — a
+ * zoom, a y-axis rescale and a window resize all move it without changing
+ * anything the card's owner can observe.
+ */
+describe('ExemplarDot position reporting', () => {
+ const renderAt = (
+ cx: number,
+ cy: number,
+ props: Partial>,
+ ) =>
+ render(
+
+ ,
+ );
+
+ it('reports its position when it is the active marker', () => {
+ const onPositionChange = jest.fn();
+ renderAt(10, 20, { isActive: true, onPositionChange });
+
+ expect(onPositionChange).toHaveBeenCalledWith(10, 20);
+ });
+
+ it('reports again when recharts moves it', () => {
+ const onPositionChange = jest.fn();
+ const { rerender } = renderAt(10, 20, {
+ isActive: true,
+ onPositionChange,
+ });
+ onPositionChange.mockClear();
+
+ rerender(
+
+ ,
+ );
+
+ expect(onPositionChange).toHaveBeenCalledWith(80, 140);
+ });
+
+ it('stays quiet when it is not the active marker', () => {
+ // Otherwise every marker reports on every frame and nothing reads the rest.
+ const onPositionChange = jest.fn();
+ renderAt(10, 20, { isActive: false, onPositionChange });
+
+ expect(onPositionChange).not.toHaveBeenCalled();
+ });
+
+ it('does not report before recharts supplies coordinates', () => {
+ const onPositionChange = jest.fn();
+ render(
+
+ ,
+ );
+
+ expect(onPositionChange).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/app/src/components/Exemplars/__tests__/ExemplarHoverCard.test.tsx b/packages/app/src/components/Exemplars/__tests__/ExemplarHoverCard.test.tsx
new file mode 100644
index 0000000000..2e81e22fb1
--- /dev/null
+++ b/packages/app/src/components/Exemplars/__tests__/ExemplarHoverCard.test.tsx
@@ -0,0 +1,89 @@
+import { Exemplar } from '@hyperdx/common-utils/dist/types';
+
+import { ExemplarHoverCard } from '@/components/Exemplars/ExemplarHoverCard';
+
+const exemplar: Exemplar = {
+ timestamp: 1704067200000, // 2024-01-01T00:00:00Z
+ value: 1234.5,
+ traceId: 'abc123def456789012345678',
+};
+
+const props = {
+ hovered: { exemplar, x: 10, y: 20 },
+ isLoading: false,
+ traceSourceConfigured: false,
+ onInspect: jest.fn(),
+ onMouseEnter: jest.fn(),
+ onMouseLeave: jest.fn(),
+};
+
+/**
+ * The marker's drawn position is deliberately not the truth: clampExemplarY pins
+ * it into the y-domain and clampExemplarX into the x-domain. Both clamps are
+ * justified in code by the card reporting the real value and time, so these are
+ * the assertions that keep that justification honest.
+ */
+describe('ExemplarHoverCard', () => {
+ it('always shows the exemplar value and time', () => {
+ const { getByText } = renderWithMantine( & { timestamp: number; value: number },
+): Exemplar => ({
+ traceId: `t-${over.timestamp}-${over.value}`,
+ ...over,
+});
+
+const MINUTE = 60_000;
+
+describe('computeExemplarPoints', () => {
+ const opts = { maxExemplars: 12, granularity: '1 minute' };
+
+ it('returns [] for empty/undefined', () => {
+ expect(computeExemplarPoints(undefined, opts)).toEqual([]);
+ expect(computeExemplarPoints([], opts)).toEqual([]);
+ });
+
+ it('maps timestamp (ms) to seconds on the x-axis and value to y', () => {
+ const [p] = computeExemplarPoints(
+ [ex({ timestamp: 1_700_000_000_000, value: 42 })],
+ opts,
+ );
+ expect(p.x).toBe(1_700_000_000); // ms -> s
+ expect(p.y).toBe(42);
+ });
+
+ it('unlimited (maxExemplars <= 0): keeps all, deduped by trace id + timestamp', () => {
+ const points = computeExemplarPoints(
+ [
+ ex({ traceId: 'a', timestamp: 1000, value: 1 }),
+ ex({ traceId: 'a', timestamp: 1000, value: 1 }), // dup
+ ex({ traceId: 'b', timestamp: 2000, value: 2 }),
+ ],
+ { ...opts, maxExemplars: 0 },
+ );
+ expect(points).toHaveLength(2);
+ });
+
+ it('skips exemplars with a non-finite value', () => {
+ const points = computeExemplarPoints(
+ [ex({ timestamp: 1000, value: NaN }), ex({ timestamp: 2000, value: 3 })],
+ { ...opts, maxExemplars: 0 },
+ );
+ expect(points).toHaveLength(1);
+ expect(points[0].y).toBe(3);
+ });
+
+ it('skips exemplars with a non-finite timestamp', () => {
+ // timestamp feeds both the bucket key and the x coordinate, so a non-finite
+ // one collapses every affected exemplar into a single "@NaN" bucket and
+ // reaches recharts as a NaN x, rendering an invalid SVG path.
+ const points = computeExemplarPoints(
+ [
+ ex({ timestamp: NaN, value: 5 }),
+ ex({ timestamp: Infinity, value: 6 }),
+ ex({ timestamp: 2000, value: 3 }),
+ ],
+ { ...opts, maxExemplars: 0 },
+ );
+ expect(points).toHaveLength(1);
+ expect(points[0].x).toBe(2);
+ expect(Number.isFinite(points[0].x)).toBe(true);
+ });
+
+ it('keeps the highest-value exemplar of a bucket', () => {
+ // Two exemplars in the same bucket, values too close to be separate
+ // samples -> only the max is plotted.
+ const points = computeExemplarPoints(
+ [
+ ex({ traceId: 'low', timestamp: 1000, value: 8 }),
+ ex({ traceId: 'high', timestamp: 1000, value: 9 }),
+ ],
+ opts,
+ );
+ expect(points).toHaveLength(1);
+ expect(points[0].y).toBe(9);
+ });
+
+ it('also keeps a value more than 2σ below the bucket max', () => {
+ // A bucket holding a cluster of typical traces plus one big outlier should
+ // surface both ends, so the overlay shows the spread and not just a max
+ // envelope. Values within the cluster stay collapsed to one marker.
+ const typical = Array.from({ length: 10 }, (_, i) =>
+ ex({ traceId: `typical-${i}`, timestamp: 1000, value: 10 + i }),
+ );
+ const points = computeExemplarPoints(
+ [ex({ traceId: 'slow', timestamp: 1000, value: 1000 }), ...typical],
+ opts,
+ );
+ expect(points.map(p => p.y)).toEqual([1000, 19]);
+ });
+
+ it('separates buckets by series (groupKey) so distinct series both survive', () => {
+ // Same time bucket, different groupKey -> both kept (one per series).
+ const points = computeExemplarPoints(
+ [
+ ex({ traceId: 'a', timestamp: 1000, value: 5, groupKey: 'svc=a' }),
+ ex({ traceId: 'b', timestamp: 1000, value: 5, groupKey: 'svc=b' }),
+ ],
+ opts,
+ );
+ expect(points).toHaveLength(2);
+ });
+
+ it('buckets at the chart granularity, not at a budget-derived width', () => {
+ // Adjacent minutes are distinct buckets even though the budget is far from
+ // exhausted, so a marker always lands in the bucket it explains.
+ const points = computeExemplarPoints(
+ [
+ ex({ traceId: 'a', timestamp: 0, value: 1 }),
+ ex({ traceId: 'b', timestamp: MINUTE, value: 2 }),
+ ex({ traceId: 'c', timestamp: 2 * MINUTE, value: 3 }),
+ ],
+ opts,
+ );
+ expect(points).toHaveLength(3);
+ });
+
+ it('spreads the marker budget across the range instead of over the spikes', () => {
+ // 20 buckets, budget of 4. The slowest traces are all bunched at the start;
+ // a value-ranked cut would return only those and leave the rest empty.
+ const exemplars = Array.from({ length: 20 }, (_, i) =>
+ ex({
+ traceId: `t${i}`,
+ timestamp: i * MINUTE,
+ value: i < 5 ? 1000 + i : 1,
+ }),
+ );
+ const points = computeExemplarPoints(exemplars, {
+ ...opts,
+ maxExemplars: 4,
+ });
+ expect(points.length).toBeLessThanOrEqual(4);
+ const times = points.map(p => p.x).sort((a, b) => a - b);
+ // Markers reach the far end of the range, not just the leading spike.
+ expect(times[times.length - 1]).toBeGreaterThanOrEqual(
+ (15 * MINUTE) / 1000,
+ );
+ });
+
+ it('renders 2σ companions instead of spending the whole budget on rank 0', () => {
+ // The regression this guards: taking one window per budget slot fills the
+ // budget on every bucket's maximum, so the 2σ sampling never places a
+ // second marker and the overlay is the max envelope it exists to replace.
+ // 30 buckets, budget 12 — the ordinary case (more buckets than budget).
+ // A realistic latency shape: mostly fast requests with one slow outlier per
+ // bucket. (σ is measured across the whole set, so a 50/50 bimodal split
+ // would push 2σ above the in-bucket gap and legitimately keep one marker.)
+ const exemplars = Array.from({ length: 30 }, (_, i) => [
+ ex({ traceId: `slow-${i}`, timestamp: i * MINUTE, value: 5000 }),
+ ...Array.from({ length: 10 }, (_, j) =>
+ ex({
+ traceId: `typical-${i}-${j}`,
+ timestamp: i * MINUTE + j + 1,
+ value: 10 + j,
+ }),
+ ),
+ ]).flat();
+
+ const points = computeExemplarPoints(exemplars, opts);
+
+ // At least one bucket contributed both its peak and its typical trace.
+ const byBucket = new Map();
+ for (const p of points) {
+ const bucket = Math.floor(p.exemplar.timestamp / MINUTE);
+ byBucket.set(bucket, (byBucket.get(bucket) ?? 0) + 1);
+ }
+ expect(Math.max(...byBucket.values())).toBeGreaterThan(1);
+ // A typical trace made it onto the chart, not just the bucket maxima.
+ expect(points.some(p => p.y < 100)).toBe(true);
+ expect(points.length).toBeLessThanOrEqual(12);
+ });
+
+ it('picks the peak of each window, not a fixed stride', () => {
+ // Spikes sit at minutes 3 and 8. A stride of 5 would sample minutes 0 and 5
+ // and miss both — on a latency chart those are the only markers worth
+ // having.
+ const exemplars = Array.from({ length: 10 }, (_, i) =>
+ ex({
+ traceId: `t${i}`,
+ timestamp: i * MINUTE,
+ value: i === 3 || i === 8 ? 5000 : 10,
+ }),
+ );
+ const points = computeExemplarPoints(exemplars, {
+ ...opts,
+ maxExemplars: 2,
+ });
+ expect(points.map(p => p.x)).toEqual([
+ (3 * MINUTE) / 1000,
+ (8 * MINUTE) / 1000,
+ ]);
+ });
+
+ it('never returns more than the marker budget', () => {
+ const exemplars = Array.from({ length: 50 }, (_, i) =>
+ ex({ traceId: `t${i}`, timestamp: i * MINUTE, value: i * 100 }),
+ );
+ expect(
+ computeExemplarPoints(exemplars, { ...opts, maxExemplars: 7 }).length,
+ ).toBeLessThanOrEqual(7);
+ });
+
+ it('treats a fractional or non-finite budget as a usable one', () => {
+ // maxExemplars comes from a team setting; a value below 1 used to produce
+ // zero windows and silently empty the overlay.
+ const exemplars = Array.from({ length: 5 }, (_, i) =>
+ ex({ traceId: `t${i}`, timestamp: i * MINUTE, value: i + 1 }),
+ );
+ expect(
+ computeExemplarPoints(exemplars, { ...opts, maxExemplars: 0.5 }).length,
+ ).toBeGreaterThan(0);
+ expect(
+ computeExemplarPoints(exemplars, { ...opts, maxExemplars: NaN }).length,
+ ).toBeGreaterThan(0);
+ });
+
+ it('drops non-finite values so one Infinity cannot disable 2σ sampling', () => {
+ // Infinity used to pass the filter, make the standard deviation NaN, and
+ // switch the spread rule off for the entire chart.
+ const points = computeExemplarPoints(
+ [
+ ex({ traceId: 'inf', timestamp: 1000, value: Infinity }),
+ ex({ traceId: 'ok', timestamp: 1000, value: 5 }),
+ ],
+ opts,
+ );
+ expect(points.map(p => p.y)).toEqual([5]);
+ });
+});
+
+describe('computeExemplarPoints window split threshold', () => {
+ // The split reserved a quarter of the budget for 2-sigma companions, but it also
+ // GATED on that reduced number — so 12 populated buckets at the default budget of
+ // 12 rendered 9 markers and left three buckets bare.
+ it('emits one marker per bucket when the buckets fit the budget', () => {
+ const exemplars: Exemplar[] = Array.from({ length: 12 }, (_, i) => ({
+ timestamp: i * 60_000,
+ value: 100 + i,
+ traceId: `t${i}`,
+ }));
+ const points = computeExemplarPoints(exemplars, {
+ maxExemplars: 12,
+ granularity: '1 minute',
+ });
+ expect(points).toHaveLength(12);
+ });
+
+ it('still splits when the buckets outnumber the budget', () => {
+ const exemplars: Exemplar[] = Array.from({ length: 40 }, (_, i) => ({
+ timestamp: i * 60_000,
+ value: 100 + i,
+ traceId: `t${i}`,
+ }));
+ const points = computeExemplarPoints(exemplars, {
+ maxExemplars: 12,
+ granularity: '1 minute',
+ });
+ expect(points.length).toBeLessThanOrEqual(12);
+ expect(points.length).toBeGreaterThan(1);
+ });
+});
+
+describe('computeExemplarYBounds', () => {
+ it('uses both numeric domain bounds when the axis is fitted to data', () => {
+ expect(computeExemplarYBounds([120, 480], 400)).toEqual({
+ min: 120,
+ max: 480,
+ });
+ });
+
+ it("falls back to the visible series max for an 'auto' upper bound", () => {
+ expect(computeExemplarYBounds([0, 'auto'], 400)).toEqual({
+ min: 0,
+ max: 400,
+ });
+ });
+
+ it("floors at 0 for an 'auto' lower bound", () => {
+ expect(computeExemplarYBounds(['auto', 'auto'], 400)).toEqual({
+ min: 0,
+ max: 400,
+ });
+ });
+
+ it('tolerates a non-array domain (recharts allows a function)', () => {
+ expect(computeExemplarYBounds(() => [0, 1], 400)).toEqual({
+ min: 0,
+ max: 400,
+ });
+ });
+});
+
+describe('clampExemplarY', () => {
+ it('pins an outlier to the top of the domain instead of overflowing it', () => {
+ // Without this, recharts' default ifOverflow="discard" drops the marker and
+ // the overlay silently loses the slowest trace.
+ expect(clampExemplarY(9000, { min: 0, max: 400 })).toBe(400);
+ });
+
+ it('drops a marker below a fitted axis floor rather than raising it', () => {
+ // fitYAxisToData puts the floor at the data's own minimum, which can sit well
+ // above a fast request. Raising that marker to the floor would draw it level
+ // with the slowest points — the opposite of the truth — so it is not drawn.
+ expect(clampExemplarY(12, { min: 120, max: 480 })).toBeNull();
+ });
+
+ it('keeps a marker exactly on the floor', () => {
+ expect(clampExemplarY(120, { min: 120, max: 480 })).toBe(120);
+ });
+
+ it('leaves an in-domain value untouched', () => {
+ expect(clampExemplarY(200, { min: 120, max: 480 })).toBe(200);
+ });
+
+ it('leaves the value alone when there is no numeric series data yet', () => {
+ // visibleSeriesMax is -Infinity before any data arrives.
+ expect(clampExemplarY(200, { min: 0, max: -Infinity })).toBe(200);
+ });
+
+ it('leaves the value alone for inverted bounds', () => {
+ expect(clampExemplarY(200, { min: 480, max: 120 })).toBe(200);
+ });
+});
+
+describe('clampExemplarX', () => {
+ const domain: [number, number] = [1_700_000_000, 1_700_000_060];
+ const bucket = 60; // one granularity, in chart time units (seconds)
+
+ // The domain's upper bound is the last *bucket start* when
+ // dateRangeEndInclusive is false, so an exemplar inside that final bucket sits
+ // past it and recharts' ifOverflow="discard" drops it — losing the newest
+ // window, which is the one a live investigation is watching.
+ it('nudges a marker inside the final partial bucket onto the last bucket', () => {
+ expect(clampExemplarX(1_700_000_090, domain, bucket)).toBe(1_700_000_060);
+ });
+
+ it('nudges a marker just before the domain start onto the first bucket', () => {
+ expect(clampExemplarX(1_699_999_970, domain, bucket)).toBe(1_700_000_000);
+ });
+
+ it('leaves an in-domain marker exactly where it is', () => {
+ expect(clampExemplarX(1_700_000_030, domain, bucket)).toBe(1_700_000_030);
+ });
+
+ // The regression this guards: placeholderData deliberately keeps the previous
+ // range's exemplars across a range change, so a zoom hands this markers from a
+ // wider window. Dragging those to the axis edge would draw real, clickable
+ // traces on buckets they never occurred in.
+ it('drops a marker more than one bucket past the domain end', () => {
+ expect(clampExemplarX(1_700_000_600, domain, bucket)).toBeNull();
+ });
+
+ it('drops a marker more than one bucket before the domain start', () => {
+ expect(clampExemplarX(1_699_999_000, domain, bucket)).toBeNull();
+ });
+
+ it('keeps the boundary case exactly one bucket out', () => {
+ expect(clampExemplarX(1_700_000_120, domain, bucket)).toBe(1_700_000_060);
+ expect(clampExemplarX(1_699_999_940, domain, bucket)).toBe(1_700_000_000);
+ });
+
+ it('treats a non-finite bucket width as zero tolerance', () => {
+ // An unparseable granularity must not widen the window silently.
+ expect(clampExemplarX(1_700_000_061, domain, NaN)).toBeNull();
+ expect(clampExemplarX(1_700_000_030, domain, NaN)).toBe(1_700_000_030);
+ });
+
+ it('leaves x untouched for a degenerate or inverted domain', () => {
+ // Better an occasionally-discarded marker than one pinned to a meaningless
+ // position — same rule as clampExemplarY.
+ expect(clampExemplarX(42, [NaN, 100], bucket)).toBe(42);
+ expect(clampExemplarX(42, [100, NaN], bucket)).toBe(42);
+ expect(clampExemplarX(42, [100, 0], bucket)).toBe(42);
+ });
+});
diff --git a/packages/app/src/components/Exemplars/__tests__/promqlSeriesLabels.test.ts b/packages/app/src/components/Exemplars/__tests__/promqlSeriesLabels.test.ts
new file mode 100644
index 0000000000..1e27709e41
--- /dev/null
+++ b/packages/app/src/components/Exemplars/__tests__/promqlSeriesLabels.test.ts
@@ -0,0 +1,167 @@
+import {
+ labelDistinguishesSeries,
+ promqlSeriesLabelRule,
+} from '@/components/Exemplars/promqlSeriesLabels';
+
+/**
+ * The rule decides whether two raw Prometheus series are the same *plotted* line.
+ * Both failure directions are silent, so both are pinned here:
+ *
+ * - too strict (`all` when the query really did aggregate) empties the overlay;
+ * - too loose (dropping a label that really does distinguish lines) merges
+ * distinct series and renders markers against a line they don't belong to.
+ *
+ * The second is worse, so every ambiguous shape must resolve to `all`.
+ */
+describe('promqlSeriesLabelRule', () => {
+ const distinguishing = (expression: string, labels: string[]) => {
+ const rule = promqlSeriesLabelRule(expression);
+ return labels.filter(l => labelDistinguishesSeries(rule, l));
+ };
+
+ it('keeps only the `by` labels', () => {
+ // The canonical latency query: one plotted line, so `instance` must stop
+ // counting or the multi-target fan-out empties the overlay.
+ expect(
+ distinguishing(
+ 'histogram_quantile(0.95, sum(rate(http_latency_bucket[5m])) by (le))',
+ ['le', 'instance', 'pod'],
+ ),
+ ).toEqual(['le']);
+ });
+
+ it('reads the prefix form of `by` as well as the suffix form', () => {
+ expect(
+ distinguishing(
+ 'histogram_quantile(0.95, sum by (le) (rate(http_latency_bucket[5m])))',
+ ['le', 'instance'],
+ ),
+ ).toEqual(['le']);
+ });
+
+ it('excludes only the `without` labels when there is no `by`', () => {
+ // The regression this covers: a `without`-only expression previously fell
+ // through to "every label counts", so the without-spelling of the query above
+ // still dropped the whole overlay.
+ expect(
+ distinguishing(
+ 'histogram_quantile(0.95, sum(rate(http_latency_bucket[5m])) without (instance))',
+ ['le', 'instance', 'pod'],
+ ),
+ ).toEqual(['le', 'pod']);
+ });
+
+ it('treats a bare selector as fully distinguishing', () => {
+ // No aggregation: this really does draw one line per instance.
+ expect(
+ distinguishing(
+ 'histogram_quantile(0.95, rate(http_latency_bucket[5m]))',
+ ['le', 'instance'],
+ ),
+ ).toEqual(['le', 'instance']);
+ });
+
+ describe('falls back to fully-distinguishing on anything ambiguous', () => {
+ // Each of these could otherwise shrink the identity and merge real series.
+ it.each([
+ [
+ 'both by and without present',
+ 'sum(rate(a[5m])) by (le) + sum(rate(b[5m])) without (instance)',
+ ],
+ [
+ 'differing by sets (nested aggregation)',
+ 'max by (service) (histogram_quantile(0.95, sum by (le, service, pod) (rate(x[5m]))))',
+ ],
+ [
+ 'top-level arithmetic',
+ 'histogram_quantile(0.95, sum(rate(a[5m])) by (le)) * 1000',
+ ],
+ [
+ 'top-level comparison',
+ 'histogram_quantile(0.95, sum(rate(a[5m])) by (le)) > 0.5',
+ ],
+ ['two without clauses', 'sum(sum(rate(a[5m])) without (x)) without (y)'],
+ ])('%s', (_name, expression) => {
+ expect(promqlSeriesLabelRule(expression)).toEqual({ mode: 'all' });
+ });
+ });
+
+ it('ignores clause-like text inside string literals', () => {
+ // A label *value* containing `by (` must not be read as syntax.
+ expect(promqlSeriesLabelRule('rate(x{job="sum by (le)"}[5m])')).toEqual({
+ mode: 'all',
+ });
+ expect(
+ distinguishing('sum(rate(x{path="/a+b"}[5m])) by (le)', [
+ 'le',
+ 'instance',
+ ]),
+ ).toEqual(['le']);
+ });
+
+ it('does not read a rate interval or label list as division', () => {
+ // `[5m]` and the `/` in a path matcher previously risked tripping the
+ // binary-operator bail-out, which would silently disable the whole fix.
+ expect(
+ distinguishing('sum(rate(http_latency_bucket[5m])) by (le)', [
+ 'le',
+ 'instance',
+ ]),
+ ).toEqual(['le']);
+ });
+
+ // A label matcher is not a comparison. Leaving `!=` in the operator test made a
+ // routine matcher suppress the overlay on a query that does aggregate to one
+ // line — reported as a P2 fail-closed.
+ it('does not read a label matcher as a top-level operator', () => {
+ expect(
+ distinguishing(
+ 'histogram_quantile(0.95, sum(rate(http_latency_bucket{code!="200"}) by (le)))',
+ ['le', 'instance'],
+ ),
+ ).toEqual(['le']);
+ expect(
+ distinguishing('sum(rate(x{path=~"/api/.*",code!="200"}[5m])) by (le)', [
+ 'le',
+ 'instance',
+ ]),
+ ).toEqual(['le']);
+ });
+
+ // The opposite direction: a clause entry that is not a bare label name means we
+ // misread the expression, and building a key from it would match no real label
+ // and collapse every series into one group — the fail-open the module forbids.
+ it('bails out when a clause entry is not a bare label name', () => {
+ expect(
+ promqlSeriesLabelRule(
+ 'histogram_quantile(0.95, sum(rate(x_bucket[5m])) by ("le"))',
+ ),
+ ).toEqual({ mode: 'all' });
+ expect(promqlSeriesLabelRule('sum(rate(x[5m])) by (`le`)')).toEqual({
+ mode: 'all',
+ });
+ });
+
+ it('does not treat a `#` inside a label value as a comment', () => {
+ // Regression: stripping comments before strings truncated the expression at a
+ // URL fragment and ate the `by (le)` clause, suppressing the overlay.
+ expect(
+ distinguishing(
+ 'histogram_quantile(0.95, sum(rate(x_bucket{path="/v1#frag"}[5m])) by (le))',
+ ['le', 'instance'],
+ ),
+ ).toEqual(['le']);
+ });
+
+ it('ignores a comment that would otherwise hide a clause', () => {
+ // `#` starts a PromQL comment; a `by (...)` inside one is not real syntax.
+ expect(promqlSeriesLabelRule('sum(rate(x[5m])) # by (le)')).toEqual({
+ mode: 'all',
+ });
+ });
+
+ it('is fully distinguishing for an empty or missing expression', () => {
+ expect(promqlSeriesLabelRule(undefined)).toEqual({ mode: 'all' });
+ expect(promqlSeriesLabelRule('')).toEqual({ mode: 'all' });
+ });
+});
diff --git a/packages/app/src/components/Exemplars/exemplarPoints.ts b/packages/app/src/components/Exemplars/exemplarPoints.ts
new file mode 100644
index 0000000000..2797fc6a3f
--- /dev/null
+++ b/packages/app/src/components/Exemplars/exemplarPoints.ts
@@ -0,0 +1,272 @@
+import { convertGranularityToSeconds } from '@hyperdx/common-utils/dist/core/utils';
+import { Exemplar } from '@hyperdx/common-utils/dist/types';
+
+/** An exemplar plus the on-screen position of its marker, for the hover card. */
+export type PositionedExemplar = { exemplar: Exemplar; x: number; y: number };
+
+/** A single exemplar plotted on the chart: x in chart time units, y = value. */
+type ExemplarPoint = {
+ x: number;
+ y: number;
+ exemplar: Exemplar;
+ key: string;
+};
+
+function finiteOrNull(v: unknown): number | null {
+ // Finite, not merely non-NaN: a single Infinity would otherwise reach
+ // standardDeviation, make the spread NaN, and silently switch the 2σ rule off
+ // for the whole chart (`spread > 0` is false for NaN).
+ return typeof v === 'number' && Number.isFinite(v) ? v : null;
+}
+
+/** Sample standard deviation, 0 when there isn't enough data to have one. */
+function standardDeviation(values: number[]): number {
+ if (values.length < 2) return 0;
+ const mean = values.reduce((a, b) => a + b, 0) / values.length;
+ const variance =
+ values.reduce((acc, v) => acc + (v - mean) ** 2, 0) / (values.length - 1);
+ return Math.sqrt(variance);
+}
+
+/**
+ * Turn raw exemplars into plotted points, thinned to keep the chart legible.
+ *
+ * - `maxExemplars <= 0`: no thinning — every exemplar is a point (deduped by
+ * trace id + timestamp).
+ * - `maxExemplars > 0`: bucket at the chart granularity, per series
+ * (`groupKey`), then within each bucket keep the highest value plus any
+ * further value that sits more than 2σ below the last one kept (σ measured
+ * across the whole set). A busy bucket therefore contributes both a typical
+ * trace and its outlier, while a quiet one contributes a single marker —
+ * the overlay reads as the latency distribution rather than as a max
+ * envelope. If that leaves more buckets than the marker budget, the range is
+ * split into evenly spaced windows and the highest bucket in each survives —
+ * markers stay spread across the range *and* land on its peaks, where a
+ * globally value-ranked cut would bunch them all on one spike.
+ *
+ * Bucketing at the chart's own granularity (rather than a width derived from
+ * the marker budget) keeps a marker inside the bucket of the series point it
+ * explains. Same approach as Grafana's StandardDeviationSampler.
+ *
+ * Pure and side-effect free so the thinning behaviour can be unit-tested without
+ * a recharts render.
+ */
+export function computeExemplarPoints(
+ exemplars: Exemplar[] | undefined,
+ opts: {
+ maxExemplars: number;
+ granularity: string;
+ },
+): ExemplarPoint[] {
+ if (!exemplars?.length) return [];
+ const { granularity } = opts;
+ // The budget arrives from a team setting. Floor it to a whole number of
+ // markers, and treat a non-finite value as unlimited rather than as zero: a
+ // fractional value below 1 (or NaN) would otherwise make the window split
+ // produce no windows and empty the overlay even though exemplars exist.
+ const maxExemplars = !Number.isFinite(opts.maxExemplars)
+ ? 0
+ : opts.maxExemplars <= 0
+ ? 0
+ : Math.max(1, Math.floor(opts.maxExemplars));
+
+ const toPoint = (exemplar: Exemplar, value: number): ExemplarPoint => ({
+ x: exemplar.timestamp / 1000, // ms -> seconds (chart x unit)
+ y: value,
+ exemplar,
+ key: `exemplar-${exemplar.traceId}-${exemplar.timestamp}`,
+ });
+
+ const points: ExemplarPoint[] = [];
+ for (const exemplar of exemplars) {
+ const value = finiteOrNull(exemplar.value);
+ // `timestamp` needs the same guard as `value`: it feeds both the bucket key
+ // and the x coordinate, so a non-finite one collapses every affected exemplar
+ // into a single `@NaN` bucket and reaches recharts as a NaN x. The normalizers
+ // parse through ExemplarSchema (`.finite()`) so this shouldn't fire in the
+ // app, but this function is the pure, independently-tested boundary.
+ if (value != null && finiteOrNull(exemplar.timestamp) != null) {
+ points.push(toPoint(exemplar, value));
+ }
+ }
+ if (!points.length) return [];
+
+ if (maxExemplars <= 0) {
+ const all = new Map();
+ for (const p of points) all.set(p.key, p); // dedupe identical trace+time
+ return Array.from(all.values());
+ }
+
+ const bucketMs = convertGranularityToSeconds(granularity) * 1000 || 1;
+ const buckets = new Map<
+ string,
+ { bucket: number; max: number; points: ExemplarPoint[] }
+ >();
+ for (const p of points) {
+ const bucket = Math.floor(p.exemplar.timestamp / bucketMs);
+ const key = `${p.exemplar.groupKey ?? ''}@${bucket}`;
+ const existing = buckets.get(key);
+ if (existing) {
+ existing.points.push(p);
+ existing.max = Math.max(existing.max, p.y);
+ } else {
+ buckets.set(key, { bucket, max: p.y, points: [p] });
+ }
+ }
+
+ // More buckets than the budget allows: split them into evenly spaced windows
+ // and keep the most notable bucket in each. Coverage still spans the range,
+ // but the surviving markers land on the peaks — an even stride is blind to
+ // where the spikes are and skips straight past the marker you wanted.
+ //
+ // Take fewer windows than the budget so the leftover slots can hold the 2σ
+ // companions below. Filling one window per slot would exhaust the budget on
+ // rank 0 and the spread sampling would never render a second marker — the max
+ // envelope this function exists to avoid.
+ //
+ // Two different thresholds, which is the point. The split only happens when the
+ // buckets genuinely outnumber the budget; gating it on windowCount instead meant
+ // 12 populated buckets at the default budget of 12 rendered 9 markers and left
+ // three buckets bare, which is not "more buckets than the marker budget".
+ const windowCount = Math.max(1, Math.ceil(maxExemplars * 0.75));
+ const ordered = Array.from(buckets.values()).sort(
+ (a, b) => a.bucket - b.bucket,
+ );
+ const chosen =
+ ordered.length <= maxExemplars
+ ? ordered
+ : Array.from({ length: windowCount }, (_, i) =>
+ ordered
+ .slice(
+ Math.floor((i * ordered.length) / windowCount),
+ Math.floor(((i + 1) * ordered.length) / windowCount),
+ )
+ // >= so ties resolve to the *later* bucket: on a flat series every
+ // bucket max is equal, and preferring the earlier one would leave
+ // the right-hand edge of the chart bare.
+ .reduce((best, b) => (b.max >= best.max ? b : best)),
+ );
+
+ const spread = standardDeviation(points.map(p => p.y)) * 2;
+ const sampled = chosen.map(({ points: inBucket }) => {
+ const byValue = [...inBucket].sort((a, b) => b.y - a.y);
+ const kept = [byValue[0]];
+ for (const p of byValue.slice(1)) {
+ if (spread > 0 && kept[kept.length - 1].y - p.y > spread) kept.push(p);
+ }
+ return kept;
+ });
+
+ // Round-robin by rank so every surviving bucket gets its most notable marker
+ // before any bucket gets a second one.
+ const out: ExemplarPoint[] = [];
+ for (let rank = 0; out.length < maxExemplars; rank++) {
+ let placed = false;
+ for (const kept of sampled) {
+ if (rank >= kept.length) continue;
+ out.push(kept[rank]);
+ placed = true;
+ if (out.length >= maxExemplars) break;
+ }
+ if (!placed) break;
+ }
+ return out;
+}
+
+/** The y range an exemplar marker may be drawn in. */
+export type ExemplarYBounds = { min: number; max: number };
+
+/**
+ * Derive the y range a marker may be drawn in, from the domain the chart actually
+ * renders. A recharts `ReferenceDot` defaults to `ifOverflow="discard"`, so a
+ * marker outside the domain vanishes with no explanation.
+ *
+ * The two bounds are used differently by clampExemplarY, which is where the
+ * reasoning lives: `max` is a ceiling markers are pinned to, `min` is a threshold
+ * below which they are dropped. `min` is NOT a lift target — raising a fast
+ * request to a fitted floor would draw it level with the slowest ones.
+ *
+ * Note `min` goes numeric on more than "Fit Y Axis to Data": useChartScales also
+ * fits the floor whenever a legend selection is active, so isolating one series
+ * can start dropping below-floor markers too.
+ *
+ * `'auto'` bounds are resolved by recharts against the series data, so the
+ * series max is in-domain by construction and 0 is a safe floor for the
+ * non-negative durations exemplars are scoped to today.
+ */
+export function computeExemplarYBounds(
+ yAxisDomain: unknown,
+ visibleSeriesMax: number,
+): ExemplarYBounds {
+ const [lower, upper] = Array.isArray(yAxisDomain)
+ ? yAxisDomain
+ : [undefined, undefined];
+ return {
+ min: typeof lower === 'number' ? lower : 0,
+ max: typeof upper === 'number' ? upper : visibleSeriesMax,
+ };
+}
+
+/**
+ * Place an exemplar's value inside `bounds` so recharts keeps drawing it, or
+ * return null to drop it.
+ *
+ * The two directions are not equivalent, so they are handled differently.
+ *
+ * Above `max`: pinned to the top. An outlier can be many times the plotted
+ * quantile, and letting it set the axis would flatten the series into a line at
+ * the bottom. A marker at the ceiling reads as "at least this high", and the
+ * hover card carries the real number.
+ *
+ * Below `min`: dropped. With `fitYAxisToData` the floor is the data's own
+ * minimum, so it can sit well above a fast request — and raising that marker to
+ * the floor would draw it level with the slowest points, saying the opposite of
+ * the truth. There is no honest position for it on this axis, so it is not drawn.
+ *
+ * Degenerate bounds (no numeric data yet, or inverted) leave the value untouched.
+ */
+export function clampExemplarY(
+ y: number,
+ bounds: ExemplarYBounds,
+): number | null {
+ const { min, max } = bounds;
+ if (!Number.isFinite(max) || !Number.isFinite(min) || min > max) return y;
+ if (y < min) return null;
+ return Math.min(y, max);
+}
+
+/**
+ * Place an exemplar's x (chart time units) inside the rendered x-domain, or
+ * return null to drop it.
+ *
+ * `ReferenceDot` defaults to `ifOverflow="discard"`, so a marker outside the
+ * domain silently vanishes. That loses the newest window: when
+ * `dateRangeEndInclusive` is false the domain's upper bound is the *last bucket
+ * start*, so an exemplar occurring inside that final bucket sits past the bound
+ * and disappears — the window a live investigation is watching. Nudging it onto
+ * the boundary of the bucket whose series point it explains fixes that.
+ *
+ * But only within one bucket. Anything further out is not an edge case, it is a
+ * marker that belongs to a different window: `placeholderData` deliberately keeps
+ * the previous range's exemplars across a range change, so a zoom hands this
+ * function markers from the old, wider window. Pulling those to the axis edge
+ * would draw real, clickable traces on buckets they did not occur in, with
+ * nothing on screen saying so. Dropping them is the honest answer — the overlay
+ * refills when the new range's query lands.
+ *
+ * `bucketSeconds` is the tolerance, in the chart's own time units. Degenerate or
+ * inverted domains leave x untouched.
+ */
+export function clampExemplarX(
+ x: number,
+ domain: [number, number],
+ bucketSeconds: number,
+): number | null {
+ const [min, max] = domain;
+ if (!Number.isFinite(min) || !Number.isFinite(max) || min > max) return x;
+ const tolerance = Number.isFinite(bucketSeconds)
+ ? Math.max(0, bucketSeconds)
+ : 0;
+ if (x < min - tolerance || x > max + tolerance) return null;
+ return Math.min(Math.max(x, min), max);
+}
diff --git a/packages/app/src/components/Exemplars/index.ts b/packages/app/src/components/Exemplars/index.ts
new file mode 100644
index 0000000000..a59f4c38f4
--- /dev/null
+++ b/packages/app/src/components/Exemplars/index.ts
@@ -0,0 +1,10 @@
+export { ExemplarDot } from './ExemplarDot';
+export { ExemplarHoverCard } from './ExemplarHoverCard';
+export {
+ clampExemplarX,
+ clampExemplarY,
+ computeExemplarPoints,
+ computeExemplarYBounds,
+ type ExemplarYBounds,
+ type PositionedExemplar,
+} from './exemplarPoints';
diff --git a/packages/app/src/components/Exemplars/promqlSeriesLabels.ts b/packages/app/src/components/Exemplars/promqlSeriesLabels.ts
new file mode 100644
index 0000000000..837a261fac
--- /dev/null
+++ b/packages/app/src/components/Exemplars/promqlSeriesLabels.ts
@@ -0,0 +1,163 @@
+/**
+ * Working out which labels distinguish the *plotted* lines of a PromQL query.
+ *
+ * Prometheus resolves `/query_exemplars` against the raw selector, so it returns
+ * one entry per underlying series while the chart draws the aggregated result.
+ * The canonical latency query
+ * `histogram_quantile(0.95, sum(rate(x_bucket[5m])) by (le))` draws a single line
+ * but comes back split across every scrape target *and* every `le` bucket. An
+ * exemplar overlay that decides "is this one series?" from the raw label
+ * cardinality therefore sees N series and drops itself on any metric scraped from
+ * more than one pod — i.e. on essentially every real deployment.
+ *
+ * Extracted from useExemplars so this rule has its own tests: the failure it
+ * guards against is silent (an empty overlay, or worse, markers attributed to the
+ * wrong line), so its edge cases need to be pinned rather than eyeballed.
+ *
+ * This is a regex approximation, not a PromQL parse. That is a deliberate
+ * trade-off, and the direction of the approximation is what matters: when the
+ * expression is anything this cannot read confidently, it reports `all` — every
+ * label distinguishes a line — which OVER-counts series and so drops the overlay.
+ * Dropping a legitimate overlay is a visible absence; rendering markers against
+ * the wrong line is a silent lie. Always fail towards `all`.
+ */
+
+/**
+ * How to decide whether two raw series belong to the same plotted line.
+ *
+ * - `all` — no usable aggregation info; every label counts (conservative).
+ * - `keep` — only `labels` count (from a `by (...)` clause).
+ * - `drop` — every label except `labels` counts (from a `without (...)` clause).
+ */
+export type SeriesLabelRule =
+ | { mode: 'all' }
+ | { mode: 'keep'; labels: Set }
+ | { mode: 'drop'; labels: Set };
+
+const ALL: SeriesLabelRule = { mode: 'all' };
+
+// `by (a, b)` / `by(a,b)` and the `without` equivalent. PromQL allows the clause
+// either before or after the argument list (`sum by (le) (x)` and
+// `sum(x) by (le)`), and both spellings match here since we only need the label
+// list, not the position.
+const BY_CLAUSE = /\bby\s*\(([^)]*)\)/g;
+const WITHOUT_CLAUSE = /\bwithout\s*\(([^)]*)\)/g;
+
+// A top-level arithmetic or comparison operator means the plotted value is a
+// derived quantity, and the aggregation clauses we can see may belong to either
+// operand. Bail to `all` rather than guess which side governs the result.
+const BINARY_OPERATOR = /[-+*/%]|==|!=|>=|<=|>|<|\b(and|or|unless)\b/;
+
+const parseLabelList = (clause: string) =>
+ clause
+ .split(',')
+ .map(s => s.trim())
+ .filter(Boolean);
+
+const sameLabels = (a: Set, b: Set) =>
+ a.size === b.size && [...a].every(l => b.has(l));
+
+/** A bare PromQL label name. Anything else in a `by`/`without` list is not one. */
+const LABEL_NAME = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
+
+/**
+ * Strip string literals and comments so their contents cannot be read as syntax —
+ * e.g. `{job="sum by (le)"}` or `{path="/a+b"}`. Replaced with `""` rather than
+ * removed so adjacent tokens do not run together.
+ *
+ * Backticks are PromQL raw strings and `#` starts a comment; both were missed
+ * before, which let either hide a clause or an operator from the checks below.
+ */
+function stripStringLiterals(expression: string): string {
+ // Strings first, comments second. A `#` is only a comment OUTSIDE a string, so
+ // stripping comments first truncates at a `#` inside a label value (a URL
+ // fragment, say) and eats the rest of the expression — including any `by (...)`
+ // clause, which silently suppresses the overlay.
+ return expression
+ .replace(/"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|`(?:[^`\\]|\\.)*`/g, '""')
+ .replace(/#[^\n]*/g, '');
+}
+
+/**
+ * Derive the series-identity rule for `expression`.
+ *
+ * Handles the two shapes that actually occur on latency charts:
+ * - one or more `by (...)` clauses -> `keep` their intersection (a label must
+ * survive every aggregation to still distinguish a line).
+ * - a `without (...)` clause and no `by` -> `drop` those labels. Without this,
+ * `sum without (instance) (...)` — the same query the `by (le)` form expresses
+ * — would fall through to `all` and empty the overlay, which is exactly the bug
+ * this module exists to fix.
+ *
+ * Anything ambiguous reports `all`:
+ * - both `by` and `without` present (they may govern different aggregations, and
+ * subtracting one from the other can shrink the identity to nothing and merge
+ * genuinely distinct lines — a fail-open we refuse).
+ * - several `by` clauses with differing label sets, or several `without` clauses.
+ * - a top-level binary operator.
+ */
+export function promqlSeriesLabelRule(
+ expression: string | undefined,
+): SeriesLabelRule {
+ if (!expression) return ALL;
+ const src = stripStringLiterals(expression);
+
+ const byClauses = [...src.matchAll(BY_CLAUSE)].map(m => parseLabelList(m[1]));
+ const withoutClauses = [...src.matchAll(WITHOUT_CLAUSE)].map(m =>
+ parseLabelList(m[1]),
+ );
+
+ // Mixed forms are not reconcilable without knowing which aggregation each
+ // clause belongs to.
+ if (byClauses.length > 0 && withoutClauses.length > 0) return ALL;
+
+ // Remove everything that can legitimately contain an operator character before
+ // testing for a top-level operator. Label matchers matter most: `!=` and `=~`
+ // are ordinary matcher syntax, so leaving `{code!="200"}` in would read as a
+ // comparison and suppress the overlay on a query that does aggregate to one
+ // line.
+ const operatorCandidate = src
+ .replace(BY_CLAUSE, '')
+ .replace(WITHOUT_CLAUSE, '')
+ .replace(/\{[^}]*\}/g, '') // label matchers: {code!="200"}
+ .replace(/\[[^\]]*\]/g, ''); // range selectors: [5m], [1h:5m]
+ if (BINARY_OPERATOR.test(operatorCandidate)) return ALL;
+
+ // A clause entry that is not a bare label name means we misread the expression
+ // (a quoted name, say, which stripStringLiterals has already rewritten to `""`).
+ // Keeping it would build a key that matches no real label, collapsing every raw
+ // series into one group and slipping past the multiple-series guard — the exact
+ // fail-open this module promises not to have.
+ const allBare = [...byClauses, ...withoutClauses]
+ .flat()
+ .every(l => LABEL_NAME.test(l));
+ if (!allBare) return ALL;
+
+ if (byClauses.length > 0) {
+ const sets = byClauses.map(l => new Set(l));
+ // Differing `by` sets mean nested aggregations we are not confident reading.
+ if (!sets.every(s => sameLabels(s, sets[0]))) return ALL;
+ return { mode: 'keep', labels: sets[0] };
+ }
+
+ if (withoutClauses.length === 1) {
+ return { mode: 'drop', labels: new Set(withoutClauses[0]) };
+ }
+
+ return ALL;
+}
+
+/** Whether `label` distinguishes one plotted line from another under `rule`. */
+export function labelDistinguishesSeries(
+ rule: SeriesLabelRule,
+ label: string,
+): boolean {
+ switch (rule.mode) {
+ case 'keep':
+ return rule.labels.has(label);
+ case 'drop':
+ return !rule.labels.has(label);
+ case 'all':
+ return true;
+ }
+}
diff --git a/packages/app/src/config.ts b/packages/app/src/config.ts
index e9df70bf1e..3e8f0d87b6 100644
--- a/packages/app/src/config.ts
+++ b/packages/app/src/config.ts
@@ -79,3 +79,7 @@ const IS_IAC_HELPERS_ENABLED = true;
// Terraform provider to talk to. Single definition — the alerts, dashboard,
// search, and team-settings surfaces all read this one constant.
export const IS_IAC_EXPORT_ENABLED = IS_IAC_HELPERS_ENABLED && !IS_LOCAL_MODE;
+// Exemplar overlay (trace markers on time charts). Off by default while the
+// feature is being tested; set NEXT_PUBLIC_ENABLE_EXEMPLARS=true to enable.
+export const IS_EXEMPLARS_ENABLED =
+ env('NEXT_PUBLIC_ENABLE_EXEMPLARS') === 'true';
diff --git a/packages/app/src/defaults.ts b/packages/app/src/defaults.ts
index 95d151307f..d38ed01118 100644
--- a/packages/app/src/defaults.ts
+++ b/packages/app/src/defaults.ts
@@ -5,6 +5,8 @@ export const DEFAULT_SEARCH_ROW_LIMIT = 200;
export const DEFAULT_QUERY_TIMEOUT = 60; // max_execution_time, seconds
export const DEFAULT_FILTER_KEYS_FETCH_LIMIT = 100;
export const DEFAULT_SERIES_LIMIT = 100;
+// Target number of exemplar markers shown per chart (0 = unlimited).
+export const DEFAULT_MAX_EXEMPLARS = 12;
export function searchChartConfigDefaults(
team: any | undefined | null,
diff --git a/packages/app/src/hooks/useExemplars/__tests__/useExemplars.test.tsx b/packages/app/src/hooks/useExemplars/__tests__/useExemplars.test.tsx
new file mode 100644
index 0000000000..2f20003d4d
--- /dev/null
+++ b/packages/app/src/hooks/useExemplars/__tests__/useExemplars.test.tsx
@@ -0,0 +1,795 @@
+/* eslint-disable @typescript-eslint/no-unsafe-type-assertion */
+import React from 'react';
+import { EXEMPLAR_QUERY_LIMIT } from '@hyperdx/common-utils/dist/core/renderChartConfig';
+import {
+ ChartConfigWithOptDateRange,
+ DisplayType,
+ Exemplar,
+ MetricsDataType,
+ SourceKind,
+ TSource,
+} from '@hyperdx/common-utils/dist/types';
+import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
+import { act, renderHook, waitFor } from '@testing-library/react';
+
+import { prometheusApi } from '@/api';
+import { useClickhouseClient } from '@/clickhouse';
+import {
+ capExemplarsPerBucket,
+ normalizePrometheusExemplars,
+ useExemplars,
+} from '@/hooks/useExemplars';
+
+// Flipped per-test to exercise the deployment feature gate. A getter (rather
+// than a literal) so the hook reads the current value on each render.
+let isExemplarsEnabled = true;
+jest.mock('@/config', () => ({
+ get IS_EXEMPLARS_ENABLED() {
+ return isExemplarsEnabled;
+ },
+}));
+
+jest.mock('@/api', () => ({
+ __esModule: true,
+ prometheusApi: { queryExemplars: jest.fn() },
+}));
+
+jest.mock('@/clickhouse', () => ({
+ __esModule: true,
+ useClickhouseClient: jest.fn(),
+}));
+
+jest.mock('@/hooks/useMetadata', () => ({
+ __esModule: true,
+ useMetadataWithSettings: jest.fn().mockReturnValue({
+ getColumns: jest.fn().mockResolvedValue([]),
+ getMaterializedColumnsLookupTable: jest.fn().mockResolvedValue(null),
+ getColumn: jest.fn().mockResolvedValue(undefined),
+ getTableMetadata: jest.fn().mockResolvedValue({ primary_key: 'TimeUnix' }),
+ getSkipIndices: jest.fn().mockResolvedValue([]),
+ getSetting: jest.fn().mockResolvedValue(undefined),
+ isClickHouseCloud: jest.fn().mockResolvedValue(false),
+ }),
+}));
+
+jest.mock('@/source', () => ({
+ __esModule: true,
+ getDurationMsExpression: jest.fn().mockReturnValue('Duration / 1e6'),
+}));
+
+const NEVER_SETTLES = () => {
+ // Deliberately never resolves or rejects.
+};
+
+describe('normalizePrometheusExemplars', () => {
+ it('returns [] for undefined/empty input', () => {
+ expect(normalizePrometheusExemplars(undefined).exemplars).toEqual([]);
+ expect(normalizePrometheusExemplars([]).exemplars).toEqual([]);
+ });
+
+ it('maps trace/span ids, value, and seconds→ms timestamp', () => {
+ const { exemplars: result } = normalizePrometheusExemplars([
+ {
+ seriesLabels: { __name__: 'http_latency', service: 'api' },
+ exemplars: [
+ {
+ labels: { trace_id: 'abc', span_id: 'def' },
+ value: '1.5',
+ timestamp: 1700000000,
+ },
+ ],
+ },
+ ]);
+ expect(result).toEqual([
+ {
+ timestamp: 1700000000 * 1000,
+ value: 1.5,
+ traceId: 'abc',
+ spanId: 'def',
+ groupKey: 'service="api"',
+ },
+ ]);
+ });
+
+ it('accepts alternate label spellings (traceID/spanID)', () => {
+ const {
+ exemplars: [ex],
+ } = normalizePrometheusExemplars([
+ {
+ seriesLabels: {},
+ exemplars: [
+ {
+ labels: { traceID: 'xyz', spanID: 's1' },
+ value: '2',
+ timestamp: 1,
+ },
+ ],
+ },
+ ]);
+ expect(ex.traceId).toBe('xyz');
+ expect(ex.spanId).toBe('s1');
+ expect(ex.groupKey).toBeUndefined();
+ });
+
+ it('merges the per-`le` entries a histogram query returns', () => {
+ // /query_exemplars reports one entry per underlying series, so a
+ // histogram_quantile chart — a single plotted line — comes back split
+ // across its bucket series. Those are the same series to the overlay.
+ const histogramBuckets = [
+ {
+ seriesLabels: { __name__: 'http_latency_bucket', le: '0.1' },
+ exemplars: [
+ { labels: { trace_id: 'a' }, value: '0.09', timestamp: 1700000000 },
+ ],
+ },
+ {
+ seriesLabels: { __name__: 'http_latency_bucket', le: '0.5' },
+ exemplars: [
+ { labels: { trace_id: 'b' }, value: '0.4', timestamp: 1700000001 },
+ ],
+ },
+ ];
+ const { exemplars: result } = normalizePrometheusExemplars(
+ histogramBuckets,
+ 'histogram_quantile(0.99, rate(http_latency_bucket[5m]))',
+ );
+ expect(result.map(e => e.traceId)).toEqual(['a', 'b']);
+ expect(result.every(e => e.groupKey === undefined)).toBe(true);
+ });
+
+ it('keeps `le` significant when the expression does not collapse buckets', () => {
+ // `rate(x_bucket[5m])` genuinely draws one line per bucket, so merging the
+ // entries would plot markers that belong to no drawn line.
+ const perBucketLines = [
+ {
+ seriesLabels: { __name__: 'http_latency_bucket', le: '0.1' },
+ exemplars: [
+ { labels: { trace_id: 'a' }, value: '0.09', timestamp: 1700000000 },
+ ],
+ },
+ {
+ seriesLabels: { __name__: 'http_latency_bucket', le: '0.5' },
+ exemplars: [
+ { labels: { trace_id: 'b' }, value: '0.4', timestamp: 1700000001 },
+ ],
+ },
+ ];
+ expect(
+ normalizePrometheusExemplars(
+ perBucketLines,
+ 'rate(http_latency_bucket[5m])',
+ ),
+ ).toEqual({ exemplars: [], dropped: 'multiple-series' });
+ });
+
+ it('drops the overlay when entries span different metrics', () => {
+ // Both entries carry no labels beyond __name__, so a label-only identity
+ // would collapse them to one series and plot markers from an unrelated
+ // metric against the drawn line.
+ const twoMetrics = [
+ {
+ seriesLabels: { __name__: 'foo' },
+ exemplars: [
+ { labels: { trace_id: 'a' }, value: '1', timestamp: 1700000000 },
+ ],
+ },
+ {
+ seriesLabels: { __name__: 'bar' },
+ exemplars: [
+ { labels: { trace_id: 'b' }, value: '2', timestamp: 1700000001 },
+ ],
+ },
+ ];
+ expect(normalizePrometheusExemplars(twoMetrics)).toEqual({
+ exemplars: [],
+ dropped: 'multiple-series',
+ });
+ });
+
+ it('drops the overlay entirely when the query returns multiple series', () => {
+ // Exemplars are single-series only; multi-series markers can't be attributed
+ // or scaled meaningfully, so the whole set is dropped rather than rendered.
+ const multiSeries = [
+ {
+ seriesLabels: { service: 'api' },
+ exemplars: [
+ { labels: { trace_id: 'a' }, value: '1', timestamp: 1700000000 },
+ ],
+ },
+ {
+ seriesLabels: { service: 'web' },
+ exemplars: [
+ { labels: { trace_id: 'b' }, value: '2', timestamp: 1700000000 },
+ ],
+ },
+ ];
+ expect(normalizePrometheusExemplars(multiSeries)).toEqual({
+ exemplars: [],
+ dropped: 'multiple-series',
+ });
+ });
+
+ it('skips exemplars without a trace id', () => {
+ expect(
+ normalizePrometheusExemplars([
+ {
+ seriesLabels: {},
+ exemplars: [{ labels: { foo: 'bar' }, value: '1', timestamp: 1 }],
+ },
+ ]).exemplars,
+ ).toEqual([]);
+ });
+
+ // The canonical latency query. /query_exemplars resolves the raw selector, so
+ // it returns one entry per scrape target *and* per `le` bucket, while the chart
+ // draws a single aggregated line. Keying the single-series guard on the full
+ // selector label set therefore emptied the overlay on any metric scraped from
+ // more than one instance — i.e. essentially always.
+ it('merges entries that differ only by labels the aggregation drops', () => {
+ const acrossInstances = [
+ {
+ seriesLabels: {
+ __name__: 'http_latency_bucket',
+ le: '0.1',
+ instance: 'pod-a',
+ },
+ exemplars: [
+ { labels: { trace_id: 'a' }, value: '0.09', timestamp: 1700000000 },
+ ],
+ },
+ {
+ seriesLabels: {
+ __name__: 'http_latency_bucket',
+ le: '0.5',
+ instance: 'pod-b',
+ },
+ exemplars: [
+ { labels: { trace_id: 'b' }, value: '0.4', timestamp: 1700000001 },
+ ],
+ },
+ ];
+ const { exemplars, dropped } = normalizePrometheusExemplars(
+ acrossInstances,
+ 'histogram_quantile(0.95, sum(rate(http_latency_bucket[5m])) by (le))',
+ );
+ expect(dropped).toBeUndefined();
+ expect(exemplars.map(e => e.traceId)).toEqual(['a', 'b']);
+ // `le` is collapsed by histogram_quantile and `instance` by the `by (le)`,
+ // so nothing distinguishes the plotted line.
+ expect(exemplars.every(e => e.groupKey === undefined)).toBe(true);
+ });
+
+ it('still drops the overlay for labels the aggregation keeps', () => {
+ // `by (le, service)` draws one line per service, so these are genuinely two
+ // plotted series and their markers can't be attributed.
+ const twoServices = [
+ {
+ seriesLabels: {
+ __name__: 'http_latency_bucket',
+ le: '0.1',
+ service: 'api',
+ instance: 'pod-a',
+ },
+ exemplars: [
+ { labels: { trace_id: 'a' }, value: '0.09', timestamp: 1700000000 },
+ ],
+ },
+ {
+ seriesLabels: {
+ __name__: 'http_latency_bucket',
+ le: '0.1',
+ service: 'web',
+ instance: 'pod-b',
+ },
+ exemplars: [
+ { labels: { trace_id: 'b' }, value: '0.4', timestamp: 1700000001 },
+ ],
+ },
+ ];
+ expect(
+ normalizePrometheusExemplars(
+ twoServices,
+ 'histogram_quantile(0.95, sum(rate(http_latency_bucket[5m])) by (le, service))',
+ ),
+ ).toEqual({ exemplars: [], dropped: 'multiple-series' });
+ });
+
+ it('merges entries under a `without` aggregation too', () => {
+ // The `without` spelling of the same canonical query. This previously fell
+ // through to "every label distinguishes a line" and dropped the whole overlay
+ // — the exact bug the `by (...)` handling was written to fix.
+ const acrossInstances = [
+ {
+ seriesLabels: {
+ __name__: 'http_latency_bucket',
+ le: '0.1',
+ instance: 'pod-a',
+ },
+ exemplars: [
+ { labels: { trace_id: 'a' }, value: '0.09', timestamp: 1700000000 },
+ ],
+ },
+ {
+ seriesLabels: {
+ __name__: 'http_latency_bucket',
+ le: '0.5',
+ instance: 'pod-b',
+ },
+ exemplars: [
+ { labels: { trace_id: 'b' }, value: '0.4', timestamp: 1700000001 },
+ ],
+ },
+ ];
+ const { exemplars, dropped } = normalizePrometheusExemplars(
+ acrossInstances,
+ 'histogram_quantile(0.95, sum(rate(http_latency_bucket[5m])) without (instance))',
+ );
+ expect(dropped).toBeUndefined();
+ expect(exemplars.map(e => e.traceId)).toEqual(['a', 'b']);
+ });
+
+ it('treats `histogram_quantile (` with a space as collapsing buckets', () => {
+ // isPromqlExemplarEligible allows whitespace before the paren, so a literal
+ // substring test here disagreed with it about one expression: the toggle
+ // allowed it, `le` stayed in the group key, and the overlay came back
+ // suppressed telling the user to aggregate to a single line they already had.
+ const { exemplars, dropped } = normalizePrometheusExemplars(
+ [
+ {
+ seriesLabels: { __name__: 'http_latency_bucket', le: '0.1' },
+ exemplars: [
+ { labels: { trace_id: 'a' }, value: '0.09', timestamp: 1700000000 },
+ ],
+ },
+ {
+ seriesLabels: { __name__: 'http_latency_bucket', le: '0.5' },
+ exemplars: [
+ { labels: { trace_id: 'b' }, value: '0.4', timestamp: 1700000001 },
+ ],
+ },
+ ],
+ 'histogram_quantile (0.95, sum(rate(http_latency_bucket[5m])) by (le))',
+ );
+ expect(dropped).toBeUndefined();
+ expect(exemplars.map(e => e.traceId)).toEqual(['a', 'b']);
+ });
+
+ it('rejects exemplars whose value or timestamp is not finite', () => {
+ // ExemplarSchema is `.finite()` on both: a NaN timestamp collapses every
+ // affected exemplar into one bucket and emits a NaN SVG coordinate.
+ const { exemplars } = normalizePrometheusExemplars([
+ {
+ seriesLabels: { __name__: 'http_latency' },
+ exemplars: [
+ { labels: { trace_id: 'ok' }, value: '1', timestamp: 1700000000 },
+ {
+ labels: { trace_id: 'bad-value' },
+ value: 'not-a-number',
+ timestamp: 1700000000,
+ },
+ {
+ labels: { trace_id: 'bad-ts' },
+ value: '1',
+ timestamp: Number.POSITIVE_INFINITY,
+ },
+ ],
+ },
+ ]);
+ expect(exemplars.map(e => e.traceId)).toEqual(['ok']);
+ });
+});
+
+describe('capExemplarsPerBucket', () => {
+ const start = new Date('2026-01-01T00:00:00Z');
+ const end = new Date('2026-01-01T01:00:00Z');
+ const at = (offsetMs: number, value: number): Exemplar => ({
+ timestamp: start.getTime() + offsetMs,
+ value,
+ traceId: `t-${offsetMs}-${value}`,
+ });
+
+ // The regression this guards: an inclusive range put a timestamp exactly on
+ // `end` one bucket past the last, giving 201 buckets. perBucket floored to 1,
+ // the total exceeded the budget, and the trailing slice trimmed a time-sorted
+ // list — so the exemplar it dropped was the newest, at the chart's right edge.
+ it('keeps the newest exemplar when the range is inclusive', () => {
+ const rangeMs = end.getTime() - start.getTime();
+ const many: Exemplar[] = [];
+ for (let i = 0; i <= EXEMPLAR_QUERY_LIMIT; i++) {
+ many.push(at(Math.round((i * rangeMs) / EXEMPLAR_QUERY_LIMIT), i));
+ }
+ const newest = many[many.length - 1];
+ const capped = capExemplarsPerBucket(many, start, end);
+
+ expect(capped.length).toBeLessThanOrEqual(EXEMPLAR_QUERY_LIMIT);
+ expect(capped.map(e => e.traceId)).toContain(newest.traceId);
+ // And the result is still chronological for the render layer.
+ expect(capped.map(e => e.timestamp)).toEqual(
+ [...capped.map(e => e.timestamp)].sort((a, b) => a - b),
+ );
+ });
+
+ it('returns the set untouched when it is already within the limit', () => {
+ const few = [at(0, 1), at(1000, 2)];
+ expect(capExemplarsPerBucket(few, start, end)).toBe(few);
+ });
+
+ it('keeps the peak of each bucket rather than a value-blind stride', () => {
+ // One slow trace early on, buried among many fast ones. A uniform index
+ // stride would drop it with ~98% probability; it is the whole reason the
+ // overlay exists.
+ const spanMs = end.getTime() - start.getTime();
+ const many = Array.from({ length: EXEMPLAR_QUERY_LIMIT * 5 }, (_, i) =>
+ at((i * spanMs) / (EXEMPLAR_QUERY_LIMIT * 5), i === 7 ? 9999 : 1),
+ );
+ const capped = capExemplarsPerBucket(many, start, end);
+ expect(capped.length).toBeLessThanOrEqual(EXEMPLAR_QUERY_LIMIT);
+ expect(capped.some(ex => ex.value === 9999)).toBe(true);
+ // Still chronological and still spanning the range.
+ expect(capped.map(ex => ex.timestamp)).toEqual(
+ [...capped.map(ex => ex.timestamp)].sort((a, b) => a - b),
+ );
+ expect(capped[capped.length - 1].timestamp).toBeGreaterThan(
+ start.getTime() + spanMs * 0.75,
+ );
+ });
+
+ it('falls back to the highest values when the range is degenerate', () => {
+ const many = Array.from({ length: EXEMPLAR_QUERY_LIMIT + 10 }, (_, i) =>
+ at(0, i),
+ );
+ const capped = capExemplarsPerBucket(many, start, start);
+ expect(capped).toHaveLength(EXEMPLAR_QUERY_LIMIT);
+ expect(Math.min(...capped.map(ex => ex.value))).toBe(10);
+ });
+});
+
+describe('useExemplars', () => {
+ const mockQuery = jest.fn();
+ const mockQueryExemplars = prometheusApi.queryExemplars as jest.Mock;
+
+ const wrapper = ({ children }: { children: React.ReactNode }) => (
+
+ {children}
+
+ );
+
+ const metricSource = { kind: SourceKind.Metric } as TSource;
+ const promqlSource = { kind: SourceKind.Promql } as TSource;
+
+ const histogramConfig = {
+ displayType: DisplayType.Line,
+ connection: 'test-connection',
+ metricTables: {
+ gauge: 'otel_metrics_gauge',
+ histogram: 'otel_metrics_histogram',
+ sum: 'otel_metrics_sum',
+ summary: 'otel_metrics_summary',
+ 'exponential histogram': 'otel_metrics_exponential_histogram',
+ },
+ from: { databaseName: 'default', tableName: '' },
+ select: [
+ {
+ aggFn: 'quantile',
+ aggCondition: '',
+ aggConditionLanguage: 'lucene',
+ valueExpression: 'Value',
+ level: 0.95,
+ metricName: 'http.server.duration',
+ metricType: MetricsDataType.Histogram,
+ },
+ ],
+ where: '',
+ whereLanguage: 'lucene',
+ timestampValueExpression: 'TimeUnix',
+ dateRange: [new Date('2025-02-12'), new Date('2025-02-14')],
+ granularity: '1 minute',
+ enableExemplars: true,
+ } as ChartConfigWithOptDateRange;
+
+ const promqlConfig = {
+ configType: 'promql',
+ displayType: DisplayType.Line,
+ connection: 'test-connection',
+ promqlExpression: 'histogram_quantile(0.95, http_latency)',
+ from: { databaseName: 'default', tableName: 'metrics' },
+ select: '',
+ where: '',
+ dateRange: [new Date('2025-02-12'), new Date('2025-02-14')],
+ granularity: '1 minute',
+ enableExemplars: true,
+ } as unknown as ChartConfigWithOptDateRange;
+
+ beforeAll(() => {
+ // The stubbed metadata can't resolve column types, which the SQL renderer
+ // warns about — expected here, and noisy.
+ jest.spyOn(console, 'warn').mockImplementation(jest.fn());
+ jest.spyOn(console, 'error').mockImplementation(jest.fn());
+ });
+ afterAll(() => {
+ jest.restoreAllMocks();
+ });
+
+ beforeEach(() => {
+ isExemplarsEnabled = true;
+ jest.clearAllMocks();
+ (useClickhouseClient as jest.Mock).mockReturnValue({ query: mockQuery });
+ });
+
+ describe('gating', () => {
+ // These assert that NO query is issued. A synchronous assertion on the first
+ // render proves nothing: `exemplars` is [] because data is undefined, and the
+ // fetch sits behind an await either way — the whole block passed with the
+ // `enabled` gate hardcoded to true. So every case now flushes the microtask
+ // queue first, and a control case proves the harness does fetch when it should.
+ const flush = () =>
+ act(async () => {
+ // Nothing to do — awaiting the act() is the point.
+ });
+
+ const renderGated = async (
+ config: Parameters[0],
+ source: Parameters[1],
+ ) => {
+ const rendered = renderHook(() => useExemplars(config, source), {
+ wrapper,
+ });
+ await flush();
+ return rendered;
+ };
+
+ it('fetches when nothing gates it (control for the cases below)', async () => {
+ const { result } = await renderGated(histogramConfig, metricSource);
+ expect(mockQuery).toHaveBeenCalled();
+ expect(result.current.exemplars).toEqual([]);
+ });
+
+ it('does not fetch when the chart has not opted in', async () => {
+ await renderGated(
+ { ...histogramConfig, enableExemplars: undefined },
+ metricSource,
+ );
+ expect(mockQuery).not.toHaveBeenCalled();
+ expect(mockQueryExemplars).not.toHaveBeenCalled();
+ });
+
+ it('does not fetch while the deployment feature flag is off', async () => {
+ isExemplarsEnabled = false;
+ await renderGated(histogramConfig, metricSource);
+ expect(mockQuery).not.toHaveBeenCalled();
+ expect(mockQueryExemplars).not.toHaveBeenCalled();
+ });
+
+ it('does not fetch for source kinds that cannot produce exemplars', async () => {
+ await renderGated(histogramConfig, {
+ kind: SourceKind.Log,
+ } as TSource);
+ expect(mockQuery).not.toHaveBeenCalled();
+ expect(mockQueryExemplars).not.toHaveBeenCalled();
+ });
+
+ it('does not fetch without a source', async () => {
+ await renderGated(histogramConfig, undefined);
+ expect(mockQuery).not.toHaveBeenCalled();
+ expect(mockQueryExemplars).not.toHaveBeenCalled();
+ });
+
+ it('does not fetch for a PromQL expression that plots no duration', async () => {
+ // promqlEligible was only ever fed an eligible expression, so the guard that
+ // stops duration markers landing on a requests/sec axis went unexercised.
+ await renderGated(
+ {
+ ...promqlConfig,
+ promqlExpression: 'rate(http_requests_total[5m])',
+ } as typeof promqlConfig,
+ promqlSource,
+ );
+ expect(mockQueryExemplars).not.toHaveBeenCalled();
+ });
+
+ it('drops the overlay when the chart draws more than one series', async () => {
+ // The rendered series count comes from the main query, not the exemplar
+ // payload — a multi-line chart must not get markers of unknown provenance.
+ // Must wait for real data: asserting [] before the query settles is true
+ // whatever the count is, which is how the first version of this test
+ // certified nothing.
+ mockQuery.mockResolvedValue({
+ json: async () => ({
+ data: [{ timestamp: '1700000000000', value: '1', traceId: 'a' }],
+ }),
+ });
+
+ const single = renderHook(
+ () => useExemplars(histogramConfig, metricSource, 1),
+ { wrapper },
+ );
+ await waitFor(() =>
+ expect(single.result.current.exemplars).toHaveLength(1),
+ );
+
+ const many = renderHook(
+ () => useExemplars(histogramConfig, metricSource, 3),
+ { wrapper },
+ );
+ await waitFor(() => expect(many.result.current.isLoading).toBe(false));
+ expect(many.result.current.exemplars).toEqual([]);
+ expect(many.result.current.dropped).toBe('multiple-series');
+ });
+ });
+
+ // placeholderData is scoped to the same chart on purpose: TanStack keeps its
+ // last-defined data on the observer, which outlives a key change, so an
+ // unscoped `prev => prev` hands the PREVIOUS metric's exemplars — real,
+ // clickable trace ids — to the new chart while isLoading and isError both say
+ // settled. Nothing exercised that comparison, so its removal was silent.
+ describe('placeholder scoping across key changes', () => {
+ const rows = (traceId: string) => ({
+ json: async () => ({
+ data: [{ timestamp: '1700000000000', value: '1', traceId }],
+ }),
+ });
+
+ it('keeps the overlay across a range-only change', async () => {
+ mockQuery.mockResolvedValue(rows('from-first-range'));
+ const { result, rerender } = renderHook(
+ ({ config }) => useExemplars(config, metricSource),
+ { wrapper, initialProps: { config: histogramConfig } },
+ );
+ await waitFor(() => expect(result.current.exemplars).toHaveLength(1));
+
+ // Same chart, later window: the markers should survive the refetch.
+ // A promise that never settles: the query is in flight for good.
+ mockQuery.mockImplementation(() => new Promise(NEVER_SETTLES));
+ rerender({
+ config: {
+ ...histogramConfig,
+ dateRange: [
+ new Date('2025-02-12T01:00:00Z'),
+ new Date('2025-02-12T02:00:00Z'),
+ ],
+ } as typeof histogramConfig,
+ });
+ expect(result.current.exemplars).toHaveLength(1);
+ });
+
+ it('blanks the overlay when the chart itself changes', async () => {
+ mockQuery.mockResolvedValue(rows('from-first-metric'));
+ const { result, rerender } = renderHook(
+ ({ config }) => useExemplars(config, metricSource),
+ { wrapper, initialProps: { config: histogramConfig } },
+ );
+ await waitFor(() => expect(result.current.exemplars).toHaveLength(1));
+
+ // A different chart (its filter changed, so it plots different data): the
+ // previous chart's traces must not be shown here.
+ // A promise that never settles: the query is in flight for good.
+ mockQuery.mockImplementation(() => new Promise(NEVER_SETTLES));
+ rerender({
+ config: {
+ ...histogramConfig,
+ where: "ServiceName = 'a-different-service'",
+ },
+ });
+ expect(result.current.exemplars).toEqual([]);
+ });
+ });
+
+ describe('metric source (ClickHouse)', () => {
+ it('maps exemplar rows and drops rows without a trace id', async () => {
+ mockQuery.mockResolvedValue({
+ json: async () => ({
+ data: [
+ {
+ timestamp: '1700000000000',
+ value: '1.5',
+ traceId: 'a',
+ spanId: 's1',
+ },
+ // No trace id → nothing to link to, must be dropped.
+ { timestamp: '1700000001000', value: '2.5', traceId: '' },
+ { timestamp: '1700000002000', value: '3.5', traceId: 'b' },
+ ],
+ }),
+ });
+
+ const { result } = renderHook(
+ () => useExemplars(histogramConfig, metricSource),
+ { wrapper },
+ );
+
+ await waitFor(() => expect(result.current.exemplars).toHaveLength(2));
+ expect(result.current.exemplars).toEqual([
+ { timestamp: 1700000000000, value: 1.5, traceId: 'a', spanId: 's1' },
+ {
+ timestamp: 1700000002000,
+ value: 3.5,
+ traceId: 'b',
+ spanId: undefined,
+ },
+ ]);
+ });
+
+ it('returns [] without querying when the config is not exemplar-eligible', async () => {
+ // A Group By makes markers unattributable, so the renderer returns no SQL.
+ const { result } = renderHook(
+ () =>
+ useExemplars(
+ { ...histogramConfig, groupBy: 'ServiceName' },
+ metricSource,
+ ),
+ { wrapper },
+ );
+ await waitFor(() => expect(result.current.isLoading).toBe(false));
+ expect(result.current.exemplars).toEqual([]);
+ expect(mockQuery).not.toHaveBeenCalled();
+ });
+ });
+
+ describe('promql source', () => {
+ it('caps the result at EXEMPLAR_QUERY_LIMIT, thinning evenly across time', async () => {
+ const exemplars = Array.from(
+ { length: EXEMPLAR_QUERY_LIMIT + 50 },
+ (_, i) => ({
+ labels: { trace_id: `t${i}` },
+ value: String(i),
+ timestamp: 1700000000 + i,
+ }),
+ );
+ mockQueryExemplars.mockResolvedValue({
+ status: 'success',
+ data: [{ seriesLabels: {}, exemplars }],
+ });
+
+ const { result } = renderHook(
+ () => useExemplars(promqlConfig, promqlSource),
+ { wrapper },
+ );
+
+ await waitFor(() =>
+ expect(result.current.exemplars).toHaveLength(EXEMPLAR_QUERY_LIMIT),
+ );
+ // Chronological and spanning the range — a value-ranked cap over the
+ // whole set would keep only the slowest traces and leave most of the
+ // range bare — while still keeping the peak within each bucket, which a
+ // uniform index stride would discard.
+ const timestamps = result.current.exemplars.map(ex => ex.timestamp);
+ expect(timestamps).toEqual([...timestamps].sort((a, b) => a - b));
+ // Markers survive at both ends of the response, so neither the start nor
+ // the tail of the range is left bare.
+ const start = 1700000000 * 1000;
+ const inputSpan = (EXEMPLAR_QUERY_LIMIT + 49) * 1000;
+ expect(timestamps[0]).toBeLessThan(start + inputSpan * 0.25);
+ expect(timestamps[timestamps.length - 1]).toBeGreaterThan(
+ start + inputSpan * 0.75,
+ );
+ // The single slowest exemplar survives the cap.
+ const slowest = EXEMPLAR_QUERY_LIMIT + 49;
+ expect(result.current.exemplars.some(ex => ex.value === slowest)).toBe(
+ true,
+ );
+ expect(mockQuery).not.toHaveBeenCalled();
+ });
+
+ it('surfaces an error when the proxy reports a non-success status', async () => {
+ mockQueryExemplars.mockResolvedValue({
+ status: 'error',
+ error: 'upstream exploded',
+ });
+
+ const { result } = renderHook(
+ () => useExemplars(promqlConfig, promqlSource),
+ { wrapper },
+ );
+
+ // The hook sets retry: 1, so allow for the retry's backoff delay.
+ await waitFor(() => expect(result.current.isError).toBe(true), {
+ timeout: 5000,
+ });
+ expect(result.current.exemplars).toEqual([]);
+ });
+ });
+});
diff --git a/packages/app/src/hooks/useExemplars/exemplarNormalize.ts b/packages/app/src/hooks/useExemplars/exemplarNormalize.ts
new file mode 100644
index 0000000000..54b8d9cf47
--- /dev/null
+++ b/packages/app/src/hooks/useExemplars/exemplarNormalize.ts
@@ -0,0 +1,224 @@
+import { EXEMPLAR_QUERY_LIMIT } from '@hyperdx/common-utils/dist/core/renderChartConfig';
+import { Exemplar, ExemplarSchema } from '@hyperdx/common-utils/dist/types';
+
+import { type PrometheusExemplarsResult } from '@/api';
+import {
+ labelDistinguishesSeries,
+ promqlSeriesLabelRule,
+ type SeriesLabelRule,
+} from '@/components/Exemplars/promqlSeriesLabels';
+
+// Native Prometheus exporters disagree on the trace/span id label name; accept
+// the common spellings.
+const TRACE_ID_LABELS = ['trace_id', 'traceID', 'traceId', 'trace.id'];
+const SPAN_ID_LABELS = ['span_id', 'spanID', 'spanId', 'span.id'];
+
+/**
+ * Whether the expression collapses a histogram's `le` buckets into one line.
+ * Only then is `le` a non-series label: `rate(x_bucket[5m])` genuinely plots one
+ * line per bucket, and treating those as one series would render markers that
+ * belong to no drawn line.
+ */
+// Whitespace-tolerant, matching isPromqlExemplarEligible. A literal substring
+// test disagreed with the eligibility gate about `histogram_quantile (0.95, ...)`:
+// the toggle allowed it, this kept `le` in the group key, and the overlay came
+// back suppressed with a notice telling the user to aggregate to a single line
+// they already had.
+const HISTOGRAM_QUANTILE_CALL = /\bhistogram_quantile\s*\(/;
+
+function collapsesHistogramBuckets(expression: string | undefined): boolean {
+ return !!expression && HISTOGRAM_QUANTILE_CALL.test(expression);
+}
+
+function pick(labels: Record, keys: string[]) {
+ for (const k of keys) {
+ if (labels[k]) return labels[k];
+ }
+ return undefined;
+}
+
+/**
+ * Stable identity for the plotted series an exemplar belongs to. `__name__` is
+ * deliberately excluded from the *label* key and counted separately by the
+ * caller — two different metrics with no other labels would otherwise both
+ * produce an empty key and merge into one overlay.
+ *
+ * `rule` restricts the key to the labels the query's aggregation actually keeps
+ * — see promqlSeriesLabelRule.
+ */
+function seriesGroupKey(
+ labels: Record,
+ ignoreLe: boolean,
+ rule: SeriesLabelRule,
+): string | undefined {
+ return (
+ Object.entries(labels)
+ .filter(
+ ([k]) =>
+ k !== '__name__' &&
+ !(ignoreLe && k === 'le') &&
+ labelDistinguishesSeries(rule, k),
+ )
+ .sort(([a], [b]) => a.localeCompare(b))
+ .map(([k, v]) => `${k}="${v}"`)
+ .join(', ') || undefined
+ );
+}
+
+/**
+ * Why an otherwise-populated exemplar overlay was suppressed, for the UI.
+ * Deliberately not exported: consumers reach it via NormalizedExemplars.dropped
+ * and compare against the literal, so exporting the alias only adds dead surface.
+ */
+type ExemplarDropReason = 'multiple-series';
+
+export type NormalizedExemplars = {
+ exemplars: Exemplar[];
+ /** Set when exemplars existed but the overlay was deliberately suppressed. */
+ dropped?: ExemplarDropReason;
+};
+
+/**
+ * Normalize a native Prometheus /query_exemplars response into the shared
+ * Exemplar shape. Exported for testing — label naming varies by exporter.
+ *
+ * Prometheus returns one entry per *underlying* series, so a
+ * `histogram_quantile(...)` query — a single plotted line — comes back split
+ * across its `le` buckets and across every scrape target. Entries that the
+ * query's aggregation collapses into one line are merged; a genuine fan-out
+ * across *plotted* series, or across different metrics, is dropped rather than
+ * rendered as unattributable markers.
+ *
+ * Every candidate is parsed through ExemplarSchema rather than coerced: the body
+ * is an untrusted upstream response that `prometheusFetch` only type-asserts, and
+ * a `Number()` of a malformed value yields NaN coordinates downstream.
+ */
+export function normalizePrometheusExemplars(
+ data: PrometheusExemplarsResult[] | undefined,
+ expression?: string,
+): NormalizedExemplars {
+ if (!data) return { exemplars: [] };
+ const ignoreLe = collapsesHistogramBuckets(expression);
+ const rule = promqlSeriesLabelRule(expression);
+ const out: Exemplar[] = [];
+ const seenSeries = new Set();
+ const seenMetrics = new Set();
+ for (const series of data) {
+ const labels = series.seriesLabels ?? {};
+ const groupKey = seriesGroupKey(labels, ignoreLe, rule);
+ for (const ex of series.exemplars ?? []) {
+ const traceId = pick(ex.labels ?? {}, TRACE_ID_LABELS);
+ if (!traceId) continue;
+ const parsed = ExemplarSchema.safeParse({
+ timestamp: ex.timestamp * 1000, // prometheus exemplar ts is unix seconds
+ value: Number(ex.value),
+ traceId,
+ spanId: pick(ex.labels ?? {}, SPAN_ID_LABELS),
+ groupKey,
+ });
+ if (!parsed.success) continue;
+ seenSeries.add(groupKey ?? '');
+ seenMetrics.add(labels.__name__ ?? '');
+ out.push(parsed.data);
+ }
+ }
+ // Exemplars are a single-series feature today: their y-position is the trace's
+ // own value on the chart's shared axis, so markers from multiple series can't
+ // be attributed or coloured yet.
+ //
+ // This payload check catches an exemplar response that spans several series or
+ // metrics. It is NOT sufficient on its own: Prometheus only returns series that
+ // carry a sampled exemplar, so it answers "how many series had exemplars", not
+ // "how many lines does the chart draw". The caller supplies the rendered series
+ // count for that — see useExemplars. Metric name is counted separately from the
+ // label key because it is excluded from that key, so two metrics with no other
+ // labels would both produce an empty key and merge.
+ if (seenSeries.size > 1 || seenMetrics.size > 1) {
+ return { exemplars: [], dropped: 'multiple-series' };
+ }
+ return { exemplars: out };
+}
+
+/**
+ * Bound a Prometheus exemplar set to EXEMPLAR_QUERY_LIMIT, mirroring what the
+ * ClickHouse scan's `LIMIT n BY bucket` does server-side: bucket by time, then
+ * keep the highest-value exemplars in each bucket.
+ *
+ * Exported for testing. A uniform index stride would be simpler but throws away
+ * the slowest traces — on a 10k-exemplar response the p99 trace the overlay
+ * exists to surface has a ~2% chance of surviving.
+ */
+export function capExemplarsPerBucket(
+ sorted: Exemplar[],
+ start: Date,
+ end: Date,
+): Exemplar[] {
+ if (sorted.length <= EXEMPLAR_QUERY_LIMIT) return sorted;
+ const rangeMs = end.getTime() - start.getTime();
+ // Degenerate range: no meaningful buckets to spread across, so fall back to
+ // the highest-value exemplars overall.
+ if (!(rangeMs > 0)) {
+ return [...sorted]
+ .sort((a, b) => b.value - a.value)
+ .slice(0, EXEMPLAR_QUERY_LIMIT)
+ .sort((a, b) => a.timestamp - b.timestamp);
+ }
+ const bucketMs = rangeMs / EXEMPLAR_QUERY_LIMIT;
+ const byBucket = new Map();
+ for (const ex of sorted) {
+ // Clamped to the last bucket. An inclusive range puts a timestamp exactly on
+ // `end` at index EXEMPLAR_QUERY_LIMIT — one past the last bucket — giving 201
+ // buckets, which drove `perBucket` to 1 and pushed the total over the budget.
+ // The trailing slice then trimmed a *time-sorted* list, so the exemplar it
+ // dropped was the newest one, at the chart's right-hand edge.
+ const bucket = Math.min(
+ EXEMPLAR_QUERY_LIMIT - 1,
+ Math.max(0, Math.floor((ex.timestamp - start.getTime()) / bucketMs)),
+ );
+ const inBucket = byBucket.get(bucket);
+ if (inBucket) inBucket.push(ex);
+ else byBucket.set(bucket, [ex]);
+ }
+ const perBucket = Math.max(
+ 1,
+ Math.floor(EXEMPLAR_QUERY_LIMIT / byBucket.size),
+ );
+ return (
+ Array.from(byBucket.keys())
+ .sort((a, b) => a - b)
+ .flatMap(k =>
+ [...byBucket.get(k)!]
+ .sort((a, b) => b.value - a.value)
+ .slice(0, perBucket),
+ )
+ // buckets <= LIMIT after the clamp above, and perBucket = floor(LIMIT/buckets),
+ // so buckets * perBucket <= LIMIT: the slice is a backstop that no longer has
+ // anything to drop. Chronological order is restored last, after the budget is
+ // already satisfied, so it can never decide *which* exemplars survive.
+ .slice(0, EXEMPLAR_QUERY_LIMIT)
+ .sort((a, b) => a.timestamp - b.timestamp)
+ );
+}
+
+/**
+ * Map raw ClickHouse exemplar rows (renderMetricExemplarsChartConfig) →
+ * Exemplar[]. Parsed rather than coerced for the same reason as the Prometheus
+ * normalizer: a row that can't produce a finite timestamp/value is dropped
+ * instead of reaching recharts as a NaN coordinate.
+ */
+export function mapClickhouseExemplars(
+ rows: Record[],
+): Exemplar[] {
+ const out: Exemplar[] = [];
+ for (const r of rows) {
+ if (!r.traceId) continue;
+ const parsed = ExemplarSchema.safeParse({
+ timestamp: Number(r.timestamp),
+ value: Number(r.value),
+ traceId: String(r.traceId),
+ spanId: r.spanId ? String(r.spanId) : undefined,
+ });
+ if (parsed.success) out.push(parsed.data);
+ }
+ return out;
+}
diff --git a/packages/app/src/hooks/useExemplars/index.ts b/packages/app/src/hooks/useExemplars/index.ts
new file mode 100644
index 0000000000..4c22c45da4
--- /dev/null
+++ b/packages/app/src/hooks/useExemplars/index.ts
@@ -0,0 +1,14 @@
+/**
+ * Public surface of the exemplars hooks. Split out of a single 430-line module;
+ * consumers (and the several test files that `jest.mock('@/hooks/useExemplars')`)
+ * import from here, so the internal file layout stays free to change.
+ */
+export {
+ capExemplarsPerBucket,
+ normalizePrometheusExemplars,
+} from './exemplarNormalize';
+export { useExemplars } from './useExemplars';
+export {
+ type ExemplarTraceMeta,
+ useExemplarTraceMeta,
+} from './useExemplarTraceMeta';
diff --git a/packages/app/src/hooks/useExemplars/quantize.ts b/packages/app/src/hooks/useExemplars/quantize.ts
new file mode 100644
index 0000000000..eaf4c216f2
--- /dev/null
+++ b/packages/app/src/hooks/useExemplars/quantize.ts
@@ -0,0 +1,28 @@
+/**
+ * Quantisation of an exemplar query's time window.
+ *
+ * Its own module because the exemplar hooks are mocked wholesale in several test
+ * files (`jest.mock('@/hooks/useExemplars')`), and these are pure helpers that
+ * callers still need for real.
+ */
+
+// Live-tail charts advance `dateRange` continuously. Rounding the range to this
+// bucket keeps sub-minute ticks on one cache entry — without it every tick mints a
+// new key, empties the overlay, and force-closes the hover card the user is
+// reaching for.
+const EXEMPLAR_KEY_QUANTUM_MS = 30_000;
+
+// Floor the start and ceil the end, so the quantised window always *contains* the
+// rendered one. Rounding both ends could produce a zero-width window and served
+// markers offset from the chart's.
+//
+// Load-bearing detail: the quantised window is what gets FETCHED, not just what
+// gets keyed. Keying on it while fetching the raw range would make the cache entry
+// hold whichever raw window arrived first, so two windows sharing a key would show
+// each other's markers. Fetching the quantised window makes the entry a genuine
+// superset of every raw window that maps to it, and the render layer then trims to
+// the drawn x-domain (see clampExemplarX).
+export const quantizeStart = (d: Date) =>
+ Math.floor(d.getTime() / EXEMPLAR_KEY_QUANTUM_MS) * EXEMPLAR_KEY_QUANTUM_MS;
+export const quantizeEnd = (d: Date) =>
+ Math.ceil(d.getTime() / EXEMPLAR_KEY_QUANTUM_MS) * EXEMPLAR_KEY_QUANTUM_MS;
diff --git a/packages/app/src/hooks/useExemplars/useExemplarTraceMeta.ts b/packages/app/src/hooks/useExemplars/useExemplarTraceMeta.ts
new file mode 100644
index 0000000000..fa7d2d6adb
--- /dev/null
+++ b/packages/app/src/hooks/useExemplars/useExemplarTraceMeta.ts
@@ -0,0 +1,69 @@
+import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types';
+import { useQuery } from '@tanstack/react-query';
+
+import { useClickhouseClient } from '@/clickhouse';
+import { getDurationMsExpression } from '@/source';
+
+export type ExemplarTraceMeta = {
+ service?: string;
+ spanName?: string;
+ statusCode?: string;
+ durationMs?: number;
+ timestamp?: string;
+};
+
+/**
+ * Fetches a one-row summary of a trace (root/first span) from the given trace
+ * source, for the exemplar hover card. Enabled only while a trace id is hovered
+ * and a trace source is configured.
+ */
+export function useExemplarTraceMeta(
+ traceId: string | undefined,
+ traceSource: TSource | undefined,
+) {
+ const clickhouseClient = useClickhouseClient();
+ const isTrace = !!traceSource && traceSource.kind === SourceKind.Trace;
+
+ return useQuery({
+ queryKey: ['exemplarTraceMeta', traceId, traceSource?.id],
+ enabled: !!traceId && isTrace,
+ staleTime: 5 * 60 * 1000,
+ queryFn: async context => {
+ if (!traceId || !traceSource || traceSource.kind !== SourceKind.Trace) {
+ return null;
+ }
+ const s = traceSource;
+ const from = s.from.databaseName
+ ? `\`${s.from.databaseName}\`.\`${s.from.tableName}\``
+ : `\`${s.from.tableName}\``;
+ const traceIdExpr = s.traceIdExpression || 'TraceId';
+ const parentExpr = s.parentSpanIdExpression || 'ParentSpanId';
+ const tsExpr = s.timestampValueExpression || 'Timestamp';
+ const sql = `
+ SELECT
+ ${s.serviceNameExpression || 'ServiceName'} AS service,
+ ${s.spanNameExpression || 'SpanName'} AS spanName,
+ ${s.statusCodeExpression || 'StatusCode'} AS statusCode,
+ ${getDurationMsExpression(s)} AS durationMs,
+ ${tsExpr} AS timestamp
+ FROM ${from}
+ WHERE ${traceIdExpr} = {traceId:String}
+ ORDER BY (${parentExpr} = '') DESC, ${tsExpr} ASC
+ LIMIT 1`;
+ const resp = await clickhouseClient.query({
+ query: sql,
+ query_params: { traceId },
+ format: 'JSON',
+ abort_signal: context.signal,
+ connectionId: s.connection,
+ });
+ const json = await resp.json();
+ const row = json.data?.[0];
+ if (!row) return null;
+ return {
+ ...row,
+ durationMs: row.durationMs != null ? Number(row.durationMs) : undefined,
+ };
+ },
+ });
+}
diff --git a/packages/app/src/hooks/useExemplars/useExemplars.tsx b/packages/app/src/hooks/useExemplars/useExemplars.tsx
new file mode 100644
index 0000000000..f89ea4d2d4
--- /dev/null
+++ b/packages/app/src/hooks/useExemplars/useExemplars.tsx
@@ -0,0 +1,219 @@
+import { useMemo } from 'react';
+import { isEqual } from 'lodash';
+import {
+ isPromqlExemplarEligible,
+ renderMetricExemplarsChartConfig,
+} from '@hyperdx/common-utils/dist/core/renderChartConfig';
+import { isPromqlChartConfig } from '@hyperdx/common-utils/dist/guards';
+import {
+ ChartConfigWithOptDateRange,
+ Exemplar,
+ SourceKind,
+ TSource,
+} from '@hyperdx/common-utils/dist/types';
+import { useQuery } from '@tanstack/react-query';
+
+import { prometheusApi } from '@/api';
+import { useClickhouseClient } from '@/clickhouse';
+import { IS_EXEMPLARS_ENABLED } from '@/config';
+import {
+ capExemplarsPerBucket,
+ mapClickhouseExemplars,
+ type NormalizedExemplars,
+ normalizePrometheusExemplars,
+} from '@/hooks/useExemplars/exemplarNormalize';
+import { quantizeEnd, quantizeStart } from '@/hooks/useExemplars/quantize';
+import { useMetadataWithSettings } from '@/hooks/useMetadata';
+
+// Source kinds that can produce exemplars today: native metric/promql sources.
+// Trace-generated exemplars are added in a follow-up.
+const EXEMPLAR_SUPPORTED_KINDS: SourceKind[] = [
+ SourceKind.Metric,
+ SourceKind.Promql,
+];
+
+// Stable identity for "no exemplars". DBTimeChart forwards this straight into
+// memo(MemoChart)'s props, so a fresh [] on every render would fail the shallow
+// compare and re-render recharts for every time chart in the app on any parent
+// state change — including with the feature flag off.
+const NO_EXEMPLARS: Exemplar[] = [];
+
+// The exemplar overlay is a coarse annotation layer, not the series itself, so it
+// tolerates being a little stale in exchange for not refiring on every mount.
+const EXEMPLAR_STALE_TIME_MS = 60_000;
+
+/**
+ * Fetches exemplars for a chart in parallel with the main series query. A no-op
+ * (disabled query) unless `config.enableExemplars` is set and the source kind
+ * supports exemplars, so it adds zero cost to charts that don't use the overlay.
+ */
+export function useExemplars(
+ config: ChartConfigWithOptDateRange,
+ source: TSource | undefined,
+ /**
+ * How many series the chart actually draws, from the main query's result.
+ *
+ * The exemplar response cannot answer this: Prometheus only returns series that
+ * carry a sampled exemplar, so a genuinely multi-line chart whose buffer
+ * happened to hold one series' worth would look single-series and get markers
+ * attributed to a line they may not belong to.
+ *
+ * An absent or zero count permits the overlay. That is safe rather than
+ * deliberate: DBTimeChart does not mount the chart until it has data, so no
+ * markers reach the screen while the count is still unknown.
+ */
+ plottedSeriesCount?: number,
+) {
+ const clickhouseClient = useClickhouseClient();
+ const metadata = useMetadataWithSettings();
+
+ const isPromql = isPromqlChartConfig(config);
+ const supported = !!source && EXEMPLAR_SUPPORTED_KINDS.includes(source.kind);
+ // A PromQL expression only carries exemplars when it plots a duration — see
+ // isPromqlExemplarEligible. Same rule the PromQL editor gates its toggle on, so
+ // a config saved before the rule tightened (or written via the API) doesn't
+ // plot duration markers on a requests/sec axis.
+ const promqlEligible =
+ !isPromql || isPromqlExemplarEligible(config.promqlExpression);
+ // Global feature gate: even a config with enableExemplars set fetches nothing
+ // while the feature is disabled for the deployment.
+ // A marker's y is the trace's own value on the chart's shared axis, so with
+ // more than one line drawn there is no way to say which line a marker belongs
+ // to. Decided from the rendered series count rather than the exemplar payload —
+ // see the parameter's note.
+ const tooManySeries = plottedSeriesCount != null && plottedSeriesCount > 1;
+
+ // Everything except the series count: "the user asked for exemplars and this
+ // chart could carry them". Kept separate so the multi-series notice can fire
+ // without a fetch.
+ const wantsExemplars =
+ IS_EXEMPLARS_ENABLED &&
+ config.enableExemplars === true &&
+ supported &&
+ promqlEligible;
+
+ // No point paying for a proxy round-trip per window on a chart that can never
+ // show the result.
+ const enabled = wantsExemplars && !tooManySeries;
+
+ // `config` minus the raw dateRange. This identifies the *chart* — the metric or
+ // PromQL expression, filters, connection — independently of the window being
+ // viewed, which is what makes it safe to carry a placeholder across a range
+ // change but not across a chart change (see placeholderData below).
+ const keyConfig = useMemo(
+ () => ({ ...config, dateRange: undefined }),
+ [config],
+ );
+
+ // The window actually fetched, and the window keyed. Must be the same value —
+ // see the note on quantizeStart above.
+ const fetchRange = useMemo(
+ () =>
+ config.dateRange
+ ? ([
+ new Date(quantizeStart(config.dateRange[0])),
+ new Date(quantizeEnd(config.dateRange[1])),
+ ] as [Date, Date])
+ : undefined,
+ [config.dateRange],
+ );
+
+ const query = useQuery({
+ // The raw dateRange is replaced by its quantized form so a live-tail tick
+ // doesn't invalidate the overlay every second.
+ queryKey: ['exemplars', keyConfig, fetchRange?.map(d => d.getTime())],
+ queryFn: async context => {
+ // PromQL → native Prometheus exemplars via the API proxy.
+ if (isPromqlChartConfig(config) && fetchRange) {
+ const [startDate, endDate] = fetchRange;
+ const resp = await prometheusApi.queryExemplars(
+ {
+ query: config.promqlExpression,
+ start: startDate.getTime() / 1000,
+ end: endDate.getTime() / 1000,
+ connectionId: config.connection,
+ database: config.from?.databaseName,
+ table: config.from?.tableName,
+ },
+ context.signal,
+ );
+ if (resp.status !== 'success') {
+ throw new Error(resp.error ?? 'query_exemplars failed');
+ }
+ const { exemplars, dropped } = normalizePrometheusExemplars(
+ resp.data,
+ config.promqlExpression,
+ );
+ // Native Prometheus /query_exemplars has no result-limit parameter, so
+ // bound the set client-side to keep an unbounded upstream response from
+ // ballooning downstream thinning/render work.
+ const all = [...exemplars].sort((a, b) => a.timestamp - b.timestamp);
+ return {
+ exemplars: capExemplarsPerBucket(all, startDate, endDate),
+ dropped,
+ };
+ }
+
+ // Structured metric source → exemplars stored on the OTel metric table.
+ const exemplarSql = await renderMetricExemplarsChartConfig(
+ // Same quantized window as the key and the PromQL branch, so one cache
+ // entry means one fetched window on both backends.
+ fetchRange ? { ...config, dateRange: fetchRange } : config,
+ metadata,
+ );
+ if (!exemplarSql) return { exemplars: [] };
+
+ const resp = await clickhouseClient.query({
+ query: exemplarSql.sql,
+ query_params: exemplarSql.params,
+ format: 'JSON',
+ abort_signal: context.signal,
+ connectionId: config.connection,
+ });
+ const json = await resp.json>();
+ return { exemplars: mapClickhouseExemplars(json.data ?? []) };
+ },
+ enabled,
+ retry: 1,
+ refetchOnWindowFocus: false,
+ staleTime: EXEMPLAR_STALE_TIME_MS,
+ // Keep the previous overlay visible across a *time-range* key change (a
+ // live-tail tick or a range nudge) instead of blanking it — the main series
+ // query does the same, and blanking here would also force-close an open hover
+ // card.
+ //
+ // Scoped to the same chart on purpose. TanStack keeps its last-defined-data on
+ // the observer instance, which outlives a key change, so an unscoped
+ // `prev => prev` hands the PREVIOUS metric's exemplars to the new chart: real,
+ // clickable trace ids, clamped onto the new axes, while isLoading/isError both
+ // report settled. The user clicks a marker and lands on an unrelated trace.
+ // Comparing the chart identity means a genuine chart switch blanks the overlay
+ // (correct) while a range tick keeps it (the point of the placeholder).
+ placeholderData: (prev, prevQuery) =>
+ isEqual(prevQuery?.queryKey?.[1], keyConfig) ? prev : undefined,
+ });
+
+ return {
+ exemplars:
+ enabled && !tooManySeries
+ ? (query.data?.exemplars ?? NO_EXEMPLARS)
+ : NO_EXEMPLARS,
+ /** The overlay was asked for but suppressed — see ExemplarDropReason. */
+ dropped: tooManySeries
+ ? wantsExemplars
+ ? ('multiple-series' as const)
+ : undefined
+ : enabled
+ ? query.data?.dropped
+ : undefined,
+ isLoading: query.isLoading,
+ isError: query.isError,
+ /**
+ * The upstream failure message, when there is one. Surfaced verbatim because
+ * the API phrases these actionably (e.g. the /query_exemplars window bound
+ * tells the user to narrow the chart's range), and a generic "could not load"
+ * would throw that guidance away.
+ */
+ error: query.error instanceof Error ? query.error.message : undefined,
+ };
+}
diff --git a/packages/common-utils/package.json b/packages/common-utils/package.json
index 33d6439858..41c74ddb9a 100644
--- a/packages/common-utils/package.json
+++ b/packages/common-utils/package.json
@@ -46,7 +46,7 @@
"dev:build": "tsup && tsc --emitDeclarationOnly --declaration",
"build": "tsup",
"ci:build": "tsup",
- "lint": "npx eslint . --ext .ts --max-warnings 92",
+ "lint": "npx eslint . --ext .ts --max-warnings 93",
"lint:fix": "npx eslint . --ext .ts --fix",
"ci:lint": "yarn lint && yarn tsc --noEmit",
"ci:unit": "jest --ci --coverage",
diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts
index 28cfa17b80..dda3385b33 100644
--- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts
+++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts
@@ -2,7 +2,12 @@ import { chSql, ColumnMeta, parameterizedQueryToSql } from '@/clickhouse';
import { Metadata } from '@/core/metadata';
import {
ChartConfigWithOptDateRangeEx,
+ EXEMPLAR_QUERY_LIMIT,
+ exemplarScanBucketing,
+ isExemplarEligible,
+ isPromqlExemplarEligible,
renderChartConfig,
+ renderMetricExemplarsChartConfig,
timeFilterExpr,
} from '@/core/renderChartConfig';
import {
@@ -3582,3 +3587,521 @@ describe('renderChartConfig', () => {
// gauge / sum / histogram snapshots earlier in this file plus the
// cross-scope integration test in packages/api/src/clickhouse/__tests__.
});
+
+describe('isExemplarEligible', () => {
+ // The single rule the chart-editor toggle and the SQL renderer both delegate
+ // to; each clause below is a case where a marker can't be attributed to a
+ // point on the chart's y-axis.
+ const eligible = {
+ seriesCount: 1,
+ seriesReturnType: 'column' as const,
+ metricType: MetricsDataType.Histogram,
+ aggFn: 'quantile',
+ hasGroupBy: false,
+ };
+
+ it('accepts a single, non-ratio, non-grouped histogram series', () => {
+ expect(isExemplarEligible(eligible)).toBe(true);
+ });
+
+ it('accepts an unset seriesReturnType (defaults to a plain column series)', () => {
+ expect(
+ isExemplarEligible({ ...eligible, seriesReturnType: undefined }),
+ ).toBe(true);
+ });
+
+ it('rejects multi-series and zero-series charts', () => {
+ expect(isExemplarEligible({ ...eligible, seriesCount: 2 })).toBe(false);
+ expect(isExemplarEligible({ ...eligible, seriesCount: 0 })).toBe(false);
+ });
+
+ it('rejects a ratio chart', () => {
+ expect(isExemplarEligible({ ...eligible, seriesReturnType: 'ratio' })).toBe(
+ false,
+ );
+ });
+
+ it('rejects a grouped chart', () => {
+ expect(isExemplarEligible({ ...eligible, hasGroupBy: true })).toBe(false);
+ });
+
+ it('rejects non-histogram and missing metric types', () => {
+ expect(
+ isExemplarEligible({ ...eligible, metricType: MetricsDataType.Gauge }),
+ ).toBe(false);
+ expect(isExemplarEligible({ ...eligible, metricType: undefined })).toBe(
+ false,
+ );
+ });
+
+ it('rejects aggregations that leave the axis in another unit', () => {
+ // A count of observations or their sum is not on the same scale as one
+ // observation, so a duration marker clamped into that domain reads as a real
+ // point on it — the failure the PromQL path already guards against.
+ expect(isExemplarEligible({ ...eligible, aggFn: 'count' })).toBe(false);
+ expect(isExemplarEligible({ ...eligible, aggFn: 'sum' })).toBe(false);
+ expect(isExemplarEligible({ ...eligible, aggFn: 'count_distinct' })).toBe(
+ false,
+ );
+ expect(isExemplarEligible({ ...eligible, aggFn: 'increase' })).toBe(false);
+ });
+
+ it('accepts aggregations that stay on the observation scale', () => {
+ for (const aggFn of ['avg', 'max', 'min', 'quantile', 'none']) {
+ expect(isExemplarEligible({ ...eligible, aggFn })).toBe(true);
+ }
+ });
+
+ it('rejects a missing aggregation', () => {
+ // A caller that cannot say what it plots cannot promise a duration axis.
+ expect(isExemplarEligible({ ...eligible, aggFn: undefined })).toBe(false);
+ });
+
+ it('rejects a differently-cased metric type', () => {
+ // The comparison is by value, so a caller passing the app package's legacy
+ // MetricsDataType ('Histogram') must not silently read as eligible.
+ expect(isExemplarEligible({ ...eligible, metricType: 'Histogram' })).toBe(
+ false,
+ );
+ });
+});
+
+describe('isPromqlExemplarEligible', () => {
+ // The PromQL counterpart: an exemplar's value is a duration, so it may only be
+ // plotted on an axis that is also a duration.
+ it('accepts expressions that plot a duration', () => {
+ expect(
+ isPromqlExemplarEligible(
+ 'histogram_quantile(0.95, sum(rate(http_latency_bucket[5m])) by (le))',
+ ),
+ ).toBe(true);
+ expect(isPromqlExemplarEligible('histogram_avg(rate(latency[5m]))')).toBe(
+ true,
+ );
+ });
+
+ it('rejects a count-valued expression', () => {
+ // The regression this guards: markers valued in milliseconds clamped into a
+ // requests/sec axis, reading as real points on that scale.
+ expect(isPromqlExemplarEligible('rate(http_requests_total[5m])')).toBe(
+ false,
+ );
+ // A raw `_bucket` series is observation *counts*, not durations.
+ expect(isPromqlExemplarEligible('rate(http_latency_bucket[5m])')).toBe(
+ false,
+ );
+ expect(isPromqlExemplarEligible('sum(up)')).toBe(false);
+ });
+
+ it('rejects an empty or missing expression', () => {
+ expect(isPromqlExemplarEligible(undefined)).toBe(false);
+ expect(isPromqlExemplarEligible('')).toBe(false);
+ });
+
+ it('rejects a duration expression wrapped in arithmetic', () => {
+ // The axis is then milliseconds while the exemplars are still seconds, so the
+ // markers would plot 1000x low and clamp to the axis floor as if real.
+ expect(
+ isPromqlExemplarEligible(
+ 'histogram_quantile(0.95, sum(rate(a[5m])) by (le)) * 1000',
+ ),
+ ).toBe(false);
+ expect(
+ isPromqlExemplarEligible(
+ '1000 * histogram_quantile(0.95, sum(rate(a[5m])) by (le))',
+ ),
+ ).toBe(false);
+ });
+
+ it('rejects the function name appearing inside a label matcher', () => {
+ expect(
+ isPromqlExemplarEligible('rate(x{note="histogram_quantile("}[5m])'),
+ ).toBe(false);
+ });
+
+ it('accepts leading whitespace before the outermost call', () => {
+ expect(
+ isPromqlExemplarEligible(' histogram_quantile(0.95, rate(a[5m]))'),
+ ).toBe(true);
+ });
+});
+
+describe('exemplarScanBucketing', () => {
+ const hours = (n: number): [Date, Date] => [
+ new Date('2026-01-01T00:00:00Z'),
+ new Date(new Date('2026-01-01T00:00:00Z').getTime() + n * 3600_000),
+ ];
+
+ it('keeps the chart granularity when it fits the row budget', () => {
+ // 1 hour at 1 minute = 60 buckets, comfortably inside the 200-row budget.
+ expect(exemplarScanBucketing('1 minute', hours(1))).toEqual({
+ interval: '1 minute',
+ perBucket: 3,
+ });
+ });
+
+ it('widens the bucket when the granularity would outrun the budget', () => {
+ const { interval, perBucket } = exemplarScanBucketing(
+ '1 minute',
+ hours(48),
+ );
+ expect(interval).toBe('869 second');
+ expect(perBucket).toBe(1);
+ });
+
+ it('leaves room for the extra epoch-aligned bucket', () => {
+ // toStartOfInterval aligns to the epoch, not the range start, so an
+ // unaligned range spans one more bucket than ceil(range / bucket). Without
+ // headroom the trailing LIMIT clips the newest bucket.
+ const [start, end] = hours(48);
+ const { interval, perBucket } = exemplarScanBucketing('1 minute', [
+ start,
+ end,
+ ]);
+ const bucketSeconds = Number(interval.split(' ')[0]);
+ const rangeSeconds = (end.getTime() - start.getTime()) / 1000;
+ const worstCaseBuckets = Math.ceil(rangeSeconds / bucketSeconds) + 1;
+ expect(worstCaseBuckets * perBucket).toBeLessThanOrEqual(
+ EXEMPLAR_QUERY_LIMIT,
+ );
+ });
+
+ it('falls back safely for degenerate inputs', () => {
+ expect(exemplarScanBucketing('1 minute', undefined)).toEqual({
+ interval: '1 minute',
+ perBucket: 1,
+ });
+ const [start] = hours(1);
+ expect(exemplarScanBucketing('1 minute', [start, start])).toEqual({
+ interval: '1 minute',
+ perBucket: 1,
+ });
+ expect(exemplarScanBucketing('1 minute', [start, new Date(NaN)])).toEqual({
+ interval: '1 minute',
+ perBucket: 1,
+ });
+ });
+
+ it('rejects a granularity that is not a valid SQL interval', () => {
+ // The interval is spliced into the query as raw SQL, and granularity
+ // reaches this function from a URL param.
+ const { interval } = exemplarScanBucketing(
+ '1 minute) UNION ALL SELECT' as never,
+ hours(1),
+ );
+ expect(interval).toBe('1 minute');
+ });
+});
+
+describe('renderMetricExemplarsChartConfig', () => {
+ let mockMetadata: jest.Mocked;
+
+ beforeAll(() => {
+ jest.spyOn(console, 'warn').mockImplementation(() => {});
+ jest.spyOn(console, 'error').mockImplementation(() => {});
+ });
+ afterAll(() => {
+ jest.restoreAllMocks();
+ });
+
+ beforeEach(() => {
+ mockMetadata = {
+ getColumns: jest.fn().mockResolvedValue([]),
+ getMaterializedColumnsLookupTable: jest.fn().mockResolvedValue(null),
+ getColumn: jest.fn().mockResolvedValue(undefined),
+ getTableMetadata: jest
+ .fn()
+ .mockResolvedValue({ primary_key: 'TimeUnix' }),
+ getSkipIndices: jest.fn().mockResolvedValue([]),
+ getSetting: jest.fn().mockResolvedValue(undefined),
+ isClickHouseCloud: jest.fn().mockResolvedValue(false),
+ } as unknown as jest.Mocked;
+ });
+
+ const histogramConfig: ChartConfigWithOptDateRange = {
+ displayType: DisplayType.Line,
+ connection: 'test-connection',
+ metricTables: {
+ gauge: 'otel_metrics_gauge',
+ histogram: 'otel_metrics_histogram',
+ sum: 'otel_metrics_sum',
+ summary: 'otel_metrics_summary',
+ 'exponential histogram': 'otel_metrics_exponential_histogram',
+ },
+ from: { databaseName: 'default', tableName: '' },
+ select: [
+ {
+ aggFn: 'quantile',
+ aggCondition: '',
+ aggConditionLanguage: 'lucene',
+ valueExpression: 'Value',
+ level: 0.95,
+ metricName: 'http.server.duration',
+ metricType: MetricsDataType.Histogram,
+ },
+ ],
+ where: '',
+ whereLanguage: 'lucene',
+ timestampValueExpression: 'TimeUnix',
+ dateRange: [new Date('2025-02-12'), new Date('2025-02-14')],
+ granularity: '1 minute',
+ };
+
+ it('builds an ARRAY JOIN exemplar query against the metric-type table', async () => {
+ const generated = await renderMetricExemplarsChartConfig(
+ histogramConfig,
+ mockMetadata,
+ );
+ expect(generated).not.toBeNull();
+ const sql = parameterizedQueryToSql(generated!);
+
+ // Surfaces the exemplar columns
+ expect(sql).toContain('ARRAY JOIN');
+ expect(sql).toContain('`Exemplars.TraceId` AS ex_TraceId');
+ expect(sql).toContain('toUnixTimestamp64Milli(ex_TimeUnix)');
+ // Points at the histogram table and filters by metric name + time
+ expect(sql).toContain('otel_metrics_histogram');
+ expect(sql).toContain("MetricName = 'http.server.duration'");
+ expect(sql).toContain('TimeUnix');
+ // Drops empty trace ids and caps the result set
+ expect(sql).toContain('notEmpty(ex_TraceId)');
+ expect(sql).toContain(`LIMIT ${EXEMPLAR_QUERY_LIMIT}`);
+ });
+
+ // renderWhere bounds the row's TimeUnix, but the projected and bucketed value is
+ // the ARRAY JOINed ex_TimeUnix, and an exemplar's own timestamp falls in the
+ // collection interval before its data point's. Without an explicit ex_TimeUnix
+ // bound the scan returned pre-window exemplars and missed in-window ones whose
+ // data point landed just after the range end.
+ it('bounds the ARRAY JOINed exemplar time, not just the row time', async () => {
+ const generated = await renderMetricExemplarsChartConfig(
+ histogramConfig,
+ mockMetadata,
+ );
+ const sql = parameterizedQueryToSql(generated!);
+
+ const start = new Date('2025-02-12').getTime();
+ const end = new Date('2025-02-14').getTime();
+ expect(sql).toContain(`ex_TimeUnix >= fromUnixTimestamp64Milli(${start})`);
+ expect(sql).toContain(`ex_TimeUnix <= fromUnixTimestamp64Milli(${end})`);
+ });
+
+ it('widens the row-level bound past the range end by one interval', async () => {
+ // So an exemplar inside the window whose data point lands just after it is
+ // still reachable; the exact ex_TimeUnix bound above then trims the overshoot.
+ const generated = await renderMetricExemplarsChartConfig(
+ histogramConfig,
+ mockMetadata,
+ );
+ const sql = parameterizedQueryToSql(generated!);
+ // 1-minute granularity, so the row bound ends 60s after the range end.
+ const widened = new Date('2025-02-14').getTime() + 60_000;
+ expect(sql).toContain(`TimeUnix <= fromUnixTimestamp64Milli(${widened})`);
+ });
+
+ it('floors the row-bound widening at one minute on a fine granularity', async () => {
+ // A short window resolves to a 15s or 30s granularity, finer than a typical
+ // scrape — widening by that much would leave the newest exemplars out of reach
+ // again, which is the whole point of the widening.
+ const generated = await renderMetricExemplarsChartConfig(
+ { ...histogramConfig, granularity: '15 second' },
+ mockMetadata,
+ );
+ const sql = parameterizedQueryToSql(generated!);
+ const widened = new Date('2025-02-14').getTime() + 60_000;
+ expect(sql).toContain(`TimeUnix <= fromUnixTimestamp64Milli(${widened})`);
+ });
+
+ it('returns null without a date range', async () => {
+ // renderWhere emits no time predicate then, leaving a whole-table ARRAY JOIN.
+ const { dateRange: _dropped, ...noRange } = histogramConfig;
+ const generated = await renderMetricExemplarsChartConfig(
+ noRange as typeof histogramConfig,
+ mockMetadata,
+ );
+ expect(generated).toBeNull();
+ });
+
+ it('spends the row budget per time bucket, not on the slowest traces overall', async () => {
+ const generated = await renderMetricExemplarsChartConfig(
+ // 1 hour at 1-minute granularity: 60 buckets, so the budget stretches to
+ // several rows each.
+ {
+ ...histogramConfig,
+ dateRange: [
+ new Date('2025-02-12T00:00:00Z'),
+ new Date('2025-02-12T01:00:00Z'),
+ ],
+ },
+ mockMetadata,
+ );
+ const sql = parameterizedQueryToSql(generated!);
+ expect(sql).toContain(
+ 'toStartOfInterval(toDateTime(ex_TimeUnix), INTERVAL 1 minute)',
+ );
+ // Ordering by bucket first is what makes the trailing LIMIT a backstop
+ // rather than a value-ranked cut of the whole range.
+ expect(sql).toContain('ORDER BY bucket ASC, value DESC');
+ expect(sql).toContain('LIMIT 3 BY bucket');
+ });
+
+ it('widens the scan bucket when the granularity would outrun the row budget', async () => {
+ const generated = await renderMetricExemplarsChartConfig(
+ // 2 days at 1-minute granularity is 2880 buckets — far more than the
+ // budget covers. Widening keeps markers spread over the whole range
+ // instead of running out partway along the x-axis.
+ histogramConfig,
+ mockMetadata,
+ );
+ const sql = parameterizedQueryToSql(generated!);
+ expect(sql).toContain(
+ 'toStartOfInterval(toDateTime(ex_TimeUnix), INTERVAL 869 second)',
+ );
+ expect(sql).toContain('LIMIT 1 BY bucket');
+ });
+
+ it("scopes exemplars to the series' aggCondition so markers match the plotted line", async () => {
+ const filteredConfig = {
+ ...histogramConfig,
+ select: [
+ {
+ aggFn: 'quantile',
+ level: 0.95,
+ valueExpression: 'Value',
+ metricName: 'http.server.duration',
+ metricType: MetricsDataType.Histogram,
+ aggCondition: "ServiceName = 'api'",
+ aggConditionLanguage: 'sql',
+ },
+ ],
+ } as ChartConfigWithOptDateRange;
+ const generated = await renderMetricExemplarsChartConfig(
+ filteredConfig,
+ mockMetadata,
+ );
+ expect(generated).not.toBeNull();
+ const sql = parameterizedQueryToSql(generated!);
+ expect(sql).toContain("ServiceName = 'api'");
+ });
+
+ it('ANDs the metric-name predicate even when the chart uses OR filters', async () => {
+ const orConfig = {
+ ...histogramConfig,
+ filtersLogicalOperator: 'OR',
+ filters: [
+ { type: 'sql', condition: "ServiceName = 'api'" },
+ { type: 'sql', condition: "ServiceName = 'web'" },
+ ],
+ } as ChartConfigWithOptDateRange;
+ const generated = await renderMetricExemplarsChartConfig(
+ orConfig,
+ mockMetadata,
+ );
+ expect(generated).not.toBeNull();
+ const sql = parameterizedQueryToSql(generated!);
+ // The required metric-name check must be ANDed as its own group, never
+ // folded into the user OR group (which would let the scan match other
+ // metrics whenever a user filter matches).
+ expect(sql).toContain("AND (MetricName = 'http.server.duration')");
+ // The user filters are still present and OR'd within their own group.
+ expect(sql).toContain("ServiceName = 'api'");
+ expect(sql).toContain("ServiceName = 'web'");
+ expect(sql).not.toMatch(/OR\s*\(?\s*MetricName/);
+ });
+
+ it('returns null for a ratio config (exemplars are meaningless on a ratio axis)', async () => {
+ const ratioConfig = {
+ ...histogramConfig,
+ seriesReturnType: 'ratio',
+ select: [histogramConfig.select[0], histogramConfig.select[0]],
+ } as ChartConfigWithOptDateRange;
+ expect(
+ await renderMetricExemplarsChartConfig(ratioConfig, mockMetadata),
+ ).toBeNull();
+ });
+
+ it('returns null for a multi-series config', async () => {
+ const multiConfig = {
+ ...histogramConfig,
+ select: [histogramConfig.select[0], histogramConfig.select[0]],
+ } as ChartConfigWithOptDateRange;
+ expect(
+ await renderMetricExemplarsChartConfig(multiConfig, mockMetadata),
+ ).toBeNull();
+ });
+
+ it('returns null when the chart has a Group By', async () => {
+ // A grouped chart draws one line per group, but the exemplar scan is not
+ // group-aware: it would pool exemplars from every group into one
+ // unattributable set. Drop the overlay instead.
+ const groupedConfig = {
+ ...histogramConfig,
+ groupBy: 'ServiceName',
+ } as ChartConfigWithOptDateRange;
+ expect(
+ await renderMetricExemplarsChartConfig(groupedConfig, mockMetadata),
+ ).toBeNull();
+ });
+
+ it('returns null for a non-histogram metric (value is on a different scale)', async () => {
+ const gaugeConfig = {
+ ...histogramConfig,
+ select: [
+ {
+ aggFn: 'avg',
+ aggCondition: '',
+ aggConditionLanguage: 'lucene',
+ valueExpression: 'Value',
+ metricName: 'system.memory.usage',
+ metricType: MetricsDataType.Gauge,
+ },
+ ],
+ } as ChartConfigWithOptDateRange;
+ expect(
+ await renderMetricExemplarsChartConfig(gaugeConfig, mockMetadata),
+ ).toBeNull();
+ });
+
+ it('scans the exponential histogram table for a native-histogram metric', async () => {
+ // Exponential (OTLP) / native (Prometheus) histograms carry exemplars in
+ // the same `Exemplars.*` columns, on their own metric-type table.
+ const exponentialConfig = {
+ ...histogramConfig,
+ select: [
+ {
+ aggFn: 'quantile',
+ aggCondition: '',
+ aggConditionLanguage: 'lucene',
+ valueExpression: 'Value',
+ level: 0.95,
+ metricName: 'traces.span.metrics.duration',
+ metricType: MetricsDataType.ExponentialHistogram,
+ },
+ ],
+ } as ChartConfigWithOptDateRange;
+ const generated = await renderMetricExemplarsChartConfig(
+ exponentialConfig,
+ mockMetadata,
+ );
+ expect(generated).not.toBeNull();
+ const sql = parameterizedQueryToSql(generated!);
+ expect(sql).toContain('otel_metrics_exponential_histogram');
+ expect(sql).toContain('`Exemplars.TraceId` AS ex_TraceId');
+ });
+
+ it('returns null for a non-metric config', async () => {
+ const logConfig: ChartConfigWithOptDateRange = {
+ displayType: DisplayType.Line,
+ connection: 'test-connection',
+ from: { databaseName: 'default', tableName: 'otel_logs' },
+ select: [{ aggFn: 'count', valueExpression: '' }],
+ where: '',
+ timestampValueExpression: 'Timestamp',
+ dateRange: [new Date('2025-02-12'), new Date('2025-02-14')],
+ granularity: '1 minute',
+ };
+ expect(
+ await renderMetricExemplarsChartConfig(logConfig, mockMetadata),
+ ).toBeNull();
+ });
+});
diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts
index f111ae8298..0900839cf7 100644
--- a/packages/common-utils/src/core/renderChartConfig.ts
+++ b/packages/common-utils/src/core/renderChartConfig.ts
@@ -56,6 +56,7 @@ import {
SortSpecificationList,
SqlAstFilter,
SQLInterval,
+ SQLIntervalSchema,
} from '@/types';
/**
@@ -2328,6 +2329,356 @@ export async function renderChartConfig(
]);
}
+/** Overall cap on exemplar markers returned for a single chart, so a wide
+ * time range can't flood the chart overlay with thousands of points. */
+export const EXEMPLAR_QUERY_LIMIT = 200;
+
+/** Ceiling on rows kept per time bucket, so a single busy bucket can't eat the
+ * whole budget when the range is short. */
+const EXEMPLAR_MAX_PER_BUCKET = 4;
+
+/**
+ * Pick the bucket width and per-bucket row count for the exemplar scan so its
+ * {@link EXEMPLAR_QUERY_LIMIT} budget is spent evenly across the range, instead
+ * of being emptied into whichever buckets happen to hold the slowest traces.
+ *
+ * The scan bucket is the chart granularity, widened only if that granularity
+ * would produce more buckets than the budget can cover — one marker per plotted
+ * point is the ideal, but running out of budget a third of the way along the
+ * x-axis is worse than a coarser sample that spans it. Widening here costs no
+ * accuracy: exemplars keep their raw timestamps, and the chart re-buckets at the
+ * true granularity when it decides what to draw.
+ */
+export function exemplarScanBucketing(
+ granularity: SQLInterval,
+ dateRange: [Date, Date] | undefined,
+): { interval: SQLInterval; perBucket: number } {
+ // Re-validate rather than trust the type: granularity reaches here from a URL
+ // param, and `interval` is spliced into the query as raw SQL by timeBucketExpr.
+ const safeGranularity: SQLInterval = SQLIntervalSchema.safeParse(granularity)
+ .success
+ ? granularity
+ : '1 minute';
+ const granularitySeconds = convertGranularityToSeconds(safeGranularity);
+ const rangeSeconds = dateRange
+ ? (dateRange[1].getTime() - dateRange[0].getTime()) / 1000
+ : 0;
+ if (
+ granularitySeconds <= 0 ||
+ !Number.isFinite(rangeSeconds) ||
+ rangeSeconds <= 0
+ ) {
+ return { interval: safeGranularity, perBucket: 1 };
+ }
+
+ // Divide by LIMIT - 1, not LIMIT: toStartOfInterval aligns to epoch multiples
+ // of the interval rather than to the range start, so a range that isn't a
+ // multiple of the widened bucket spills into one extra group. Leaving room
+ // for it keeps buckets * perBucket inside the trailing LIMIT — otherwise that
+ // LIMIT clips the newest bucket, the right-hand edge of the chart.
+ const bucketSeconds = Math.max(
+ granularitySeconds,
+ Math.ceil(rangeSeconds / (EXEMPLAR_QUERY_LIMIT - 1)),
+ );
+ const buckets = Math.ceil(rangeSeconds / bucketSeconds) + 1;
+ return {
+ interval:
+ bucketSeconds === granularitySeconds
+ ? safeGranularity
+ : `${bucketSeconds} second`,
+ perBucket: Math.min(
+ EXEMPLAR_MAX_PER_BUCKET,
+ Math.max(1, Math.floor(EXEMPLAR_QUERY_LIMIT / buckets)),
+ ),
+ };
+}
+
+/**
+ * The exemplar-eligibility rule, in one place. An exemplar marker sits at a
+ * single trace's raw measurement on the chart's y-axis, so it is only
+ * attributable when the chart draws exactly one series of a compatible unit:
+ *
+ * - single series, no Group By — a marker can't be attributed (or scaled) across
+ * series, and a grouped scan pools exemplars from every group.
+ * - not a ratio — the exemplar value isn't on a ratio axis.
+ * - histogram, explicit-bucket or exponential — the exemplar value is a
+ * duration, which only shares the y-axis unit on a latency histogram
+ * (counts/gauges/rates are a different scale). Both kinds store exemplars in
+ * the same `Exemplars.*` columns, and the scan resolves its table from
+ * `metricTables[metricType]`, so neither the query nor the marker changes.
+ *
+ * The chart-editor toggle and the SQL renderer both delegate here so they can't
+ * drift apart.
+ */
+// Aggregations whose result is on the same scale as a single observation, and so
+// share an axis with `Exemplars.Value`. A quantile, average, min or max of
+// durations is itself a duration; a count is a number of observations and a sum is
+// their total, both of which put the axis in a different unit — a duration marker
+// clamped into a count domain reads as a real point on that scale. `none` is the
+// raw value, so it qualifies.
+const EXEMPLAR_COMPATIBLE_AGG_FNS = new Set([
+ 'avg',
+ 'max',
+ 'min',
+ 'quantile',
+ 'last_value',
+ 'any',
+ 'none',
+]);
+
+export function isExemplarEligible({
+ seriesCount,
+ seriesReturnType,
+ metricType,
+ aggFn,
+ hasGroupBy,
+}: {
+ seriesCount: number;
+ seriesReturnType?: 'ratio' | 'column';
+ // Deliberately `string`, and compared by value below. tsup emits this
+ // declaration without importing MetricsDataType, so any annotation naming the
+ // enum resolves against the *consumer's* scope: in packages/app it binds to
+ // that package's unrelated legacy MetricsDataType, whose values are
+ // capitalised ('Histogram'), and the build fails on the mismatch.
+ // Callers must pass a common-utils MetricsDataType value (lowercase
+ // 'histogram'); a legacy-enum value would compile but never match.
+ metricType?: string;
+ /**
+ * The series' aggregation. `string` for the same tsup reason as `metricType`.
+ * Undefined is treated as ineligible: a caller that cannot say what it plots
+ * cannot promise the axis is a duration.
+ */
+ aggFn?: string;
+ hasGroupBy: boolean;
+}): boolean {
+ return (
+ seriesCount === 1 &&
+ seriesReturnType !== 'ratio' &&
+ (metricType === MetricsDataType.Histogram ||
+ metricType === MetricsDataType.ExponentialHistogram) &&
+ aggFn != null &&
+ EXEMPLAR_COMPATIBLE_AGG_FNS.has(aggFn) &&
+ !hasGroupBy
+ );
+}
+
+/**
+ * PromQL functions whose result shares the y-axis unit with an exemplar's value
+ * (a duration). A marker's height is the linked trace's own measurement, so it
+ * is only honest on an axis measured in the same unit.
+ *
+ * Notably absent: `rate(..._bucket[5m])` and friends. A `_bucket` series holds
+ * observation *counts*, so plotting duration-valued markers on it puts, say, a
+ * 250ms trace at "250 requests/sec" — clampExemplarY would then pin it inside
+ * the axis and it would read as a real point on that scale.
+ */
+const PROMQL_DURATION_VALUED_CALL =
+ /^\s*(histogram_quantile|histogram_avg)\s*\(/;
+
+/**
+ * The PromQL counterpart to isExemplarEligible: whether the expression plots a
+ * duration, and so can carry exemplar markers on its axis. The PromQL editor's
+ * toggle and the exemplar fetch both delegate here so they can't drift apart.
+ *
+ * Deliberately narrow rather than a full parse, and deliberately erring towards
+ * `false`: a false negative hides the toggle on a chart that could have shown
+ * markers, while a false positive plots duration markers on an axis measured in
+ * something else and clamps them into it, which reads as real data. Other
+ * duration-valued shapes (`avg_over_time(latency_seconds[5m])`, a
+ * `_sum / _count` ratio) are not recognised yet — extend this pattern when one
+ * is wanted rather than loosening it to a substring test.
+ */
+export function isPromqlExemplarEligible(
+ expression: string | undefined,
+): boolean {
+ if (!expression) return false;
+ const opening = PROMQL_DURATION_VALUED_CALL.exec(expression);
+ if (!opening) return false;
+
+ // The call must span the WHOLE expression, not merely start it. Walking to the
+ // matching close paren is what distinguishes `histogram_quantile(...)` from
+ // `histogram_quantile(...) * 1000`: the latter's axis is milliseconds while its
+ // exemplars are still seconds, so the markers would plot 1000x low and clamp to
+ // the axis floor as though they were real values. Quotes are skipped so a paren
+ // inside a label matcher doesn't unbalance the count.
+ let depth = 0;
+ let quote: string | null = null;
+ for (let i = opening[0].length - 1; i < expression.length; i++) {
+ const c = expression.charAt(i);
+ if (quote) {
+ if (c === '\\') i++;
+ else if (c === quote) quote = null;
+ continue;
+ }
+ if (c === '"' || c === "'") quote = c;
+ else if (c === '(') depth++;
+ else if (c === ')') {
+ depth--;
+ // Closed the outermost call: eligible only if nothing but whitespace is
+ // left, i.e. its value is what the axis renders.
+ if (depth === 0) return expression.slice(i + 1).trim() === '';
+ }
+ }
+ // Unbalanced parens — a half-typed expression. Not eligible.
+ return false;
+}
+
+/**
+ * Builds a ClickHouse query that surfaces native exemplars stored on an OTel
+ * metric table (`Exemplars.TraceId/SpanId/Value/TimeUnix`). Returns null when
+ * the config is not a single-metric chart we can resolve a table for.
+ *
+ * Reuses `renderWhere` so the exemplar scan honors the exact same time range,
+ * metric-name, and user filters as the rendered series. Exemplars keep their own
+ * raw timestamp/value (the marker sits at the exemplar's own measurement); the
+ * bucket column exists only to spread the result set evenly across the range —
+ * `LIMIT n BY bucket` keeps the top `n` of every bucket rather than the top `n`
+ * of the whole range, which would return nothing but the spikes.
+ */
+export async function renderMetricExemplarsChartConfig(
+ chartConfig: ChartConfigWithOptDateRangeEx,
+ metadata: Metadata,
+): Promise {
+ if (
+ isRawSqlChartConfig(chartConfig) ||
+ isPromqlChartConfig(chartConfig) ||
+ !isMetricChartConfig(chartConfig) ||
+ !Array.isArray(chartConfig.select) ||
+ // Without a range, renderWhere emits no time predicate and this becomes a
+ // whole-table ARRAY JOIN plus sort, bounded only by the trailing LIMIT. The
+ // PromQL branch already refuses this case; match it.
+ chartConfig.dateRange == null
+ ) {
+ return null;
+ }
+ const { metricTables, select } = chartConfig;
+ const { metricType, metricName, metricNameSql, aggFn } = select[0] ?? {};
+ // Shared eligibility rule (single, non-ratio, non-grouped histogram series) —
+ // see isExemplarEligible. A Group By in particular would pool exemplars from
+ // every group into one unattributable set.
+ if (
+ !isExemplarEligible({
+ seriesCount: select.length,
+ seriesReturnType: chartConfig.seriesReturnType,
+ metricType,
+ aggFn,
+ hasGroupBy: isUsingGroupBy(chartConfig),
+ })
+ ) {
+ return null;
+ }
+ const table =
+ metricType && metricTables ? metricTables[metricType] : undefined;
+ if (!metricType || !metricName || !table) {
+ return null;
+ }
+
+ // Build a config that points at the concrete metric-type table and carries
+ // the metric-name predicate alongside the user filters, then let renderWhere
+ // assemble the time filter + filters exactly as the main query does. The
+ // guards above narrow chartConfig to the metric builder config, so no cast.
+ // Bucket the scan so every part of the range gets a fair shot at a marker.
+ // 'auto' is resolved here rather than left to timeBucketExpr so the bucket
+ // column and the per-bucket limit agree on the same width.
+ const granularity: SQLInterval =
+ chartConfig.granularity && chartConfig.granularity !== 'auto'
+ ? chartConfig.granularity
+ : chartConfig.dateRange
+ ? convertDateRangeToGranularityString(chartConfig.dateRange)
+ : '1 minute';
+ const { interval, perBucket } = exemplarScanBucketing(
+ granularity,
+ chartConfig.dateRange,
+ );
+ // How far an exemplar's own timestamp can trail its data point is the metric's
+ // collection interval, which we don't know here. The chart's granularity is a
+ // proxy, not the same thing — and on a short window it resolves to 15s or 30s,
+ // finer than a typical scrape, which would leave the newest exemplars out of
+ // reach again. So take the larger of the granularity and a one-minute floor.
+ // The exact ex_TimeUnix bound below plus `LIMIT n BY bucket` trim the overshoot,
+ // so erring wide is close to free. The derived bucket width is deliberately not
+ // used: exemplarScanBucketing can stretch it to many minutes on a multi-day
+ // range, which over-fetches for no benefit.
+ const COLLECTION_INTERVAL_FLOOR_MS = 60_000;
+ const collectionIntervalMs = Math.max(
+ convertGranularityToSeconds(granularity) * 1000,
+ COLLECTION_INTERVAL_FLOOR_MS,
+ );
+
+ const [rangeStart, rangeEnd] = chartConfig.dateRange;
+
+ // renderWhere bounds the *row's* TimeUnix, but the value we project and bucket
+ // is the ARRAY JOINed `ex_TimeUnix`. Those are not the same instant: an
+ // exemplar is attached to the data point that covers it, so its own timestamp
+ // falls in the collection interval *before* that row's. So the row bound is
+ // widened forward by one interval — otherwise an exemplar inside the window
+ // whose data point lands just after it is never returned — and the exact
+ // window is then applied to `ex_TimeUnix` below, which also drops the
+ // pre-window exemplars the widened row bound now lets through.
+ const rowBoundEnd = new Date(rangeEnd.getTime() + collectionIntervalMs);
+
+ const whereConfig: BuilderChartConfigWithOptDateRangeEx = {
+ ...chartConfig,
+ dateRange: [rangeStart, rowBoundEnd],
+ from: { ...chartConfig.from, tableName: table },
+ timestampValueExpression:
+ chartConfig.timestampValueExpression || DEFAULT_METRIC_TABLE_TIME_COLUMN,
+ // Keep the original select so renderWhere applies the series' aggCondition —
+ // otherwise the exemplar scan would surface traces from other series (e.g.
+ // other services/routes/tenants) that share the same metric name.
+ };
+
+ const where = await renderWhere(whereConfig, metadata);
+ const from = renderFrom({ from: whereConfig.from });
+
+ // The metric-name predicate is REQUIRED and must always be ANDed. Appending it
+ // to `chartConfig.filters` would subject it to the chart's
+ // `filtersLogicalOperator`, so an 'OR' filter group would produce
+ // `userFilterA OR userFilterB OR MetricName = ...` and let the exemplar scan
+ // surface traces from other metrics. AND it separately from the user filters.
+ const metricNameCondition = createMetricNameFilter(metricName, metricNameSql);
+
+ const bucketExpr = timeBucketExpr({
+ interval,
+ timestampValueExpression: 'ex_TimeUnix',
+ dateRange: chartConfig.dateRange,
+ alias: 'bucket',
+ });
+
+ return concatChSql(' ', [
+ chSql`SELECT
+ toUnixTimestamp64Milli(ex_TimeUnix) AS timestamp,
+ ex_Value AS value,
+ ex_TraceId AS traceId,
+ ex_SpanId AS spanId,
+ ${bucketExpr}`,
+ chSql`FROM ${from}`,
+ chSql`ARRAY JOIN
+ \`Exemplars.TimeUnix\` AS ex_TimeUnix,
+ \`Exemplars.Value\` AS ex_Value,
+ \`Exemplars.TraceId\` AS ex_TraceId,
+ \`Exemplars.SpanId\` AS ex_SpanId`,
+ chSql`WHERE ${where.sql ? where : chSql`1 = 1`} AND (${metricNameCondition}) AND notEmpty(ex_TraceId) AND ex_TimeUnix >= fromUnixTimestamp64Milli(${{ Int64: rangeStart.getTime() }}) AND ex_TimeUnix <= fromUnixTimestamp64Milli(${{ Int64: rangeEnd.getTime() }})`,
+ // Within a bucket the slowest traces are the interesting ones; across
+ // buckets we want coverage, so the per-bucket limit — not the value order —
+ // is what bounds the result set. exemplarScanBucketing keeps
+ // buckets * perBucket under the trailing LIMIT, which is a backstop only.
+ //
+ // Consequence worth knowing: this hands the client the top `perBucket`
+ // rows per bucket, so the chart's 2σ spread sampler can only choose among a
+ // bucket's slowest few — on this path the overlay is a narrow max envelope,
+ // not the full within-bucket distribution the Prometheus path can show.
+ // Widening that needs within-bucket sampling in SQL (a window function),
+ // which is a bigger change than this one.
+ chSql`ORDER BY bucket ASC, value DESC`,
+ // Raw rather than a bound parameter: `LIMIT n BY` is parsed before
+ // parameter substitution. `perBucket` (not the sibling `interval`) is a
+ // clamped integer from exemplarScanBucketing, never user input.
+ chSql`LIMIT ${{ UNSAFE_RAW_SQL: String(perBucket) }} BY bucket`,
+ chSql`LIMIT ${{ Int32: EXEMPLAR_QUERY_LIMIT }}`,
+ ]);
+}
+
// EditForm -> translateToQueriedChartConfig -> QueriedChartConfig
// renderFn(QueriedChartConfig) -> sql
// query(sql) -> data
diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts
index 154da9d24a..7f1c670ccc 100644
--- a/packages/common-utils/src/types.ts
+++ b/packages/common-utils/src/types.ts
@@ -1229,6 +1229,15 @@ const SharedChartSettingsSchema = z.object({
// number tiles have no time dimension to bucket). Other display types
// ignore the field. Kept at shared level mirroring `color` / `colorRules`.
backgroundChart: BackgroundChartSchema.optional(),
+ // Opt-in: overlay exemplar markers (individual traces linked to the series)
+ // on time charts. Sourced natively for metric/PromQL sources and generated
+ // per time bucket for trace sources. The UI gates the toggle on supported
+ // source kinds; off by default so the extra exemplar query never runs.
+ enableExemplars: z.boolean().optional(),
+ // Trace source that an exemplar's trace id resolves against — used to fetch
+ // hover metadata and to deep-link straight to the trace. When unset, the
+ // chart source's linked `traceSourceId` is used as a fallback.
+ exemplarTraceSourceId: z.string().optional(),
// Zebra striping for table tiles: when true, the renderer tints alternating
// rows so wide tables are easier to scan across. Applies to any table tile
// (builder or raw SQL); the striping is purely presentational and keys off
@@ -1366,6 +1375,30 @@ export const ChartConfigSchema = z.union([
export type ChartConfig = z.infer;
+/**
+ * A single exemplar: an individual data point overlaid on a time chart that
+ * links back to a trace. Shared shape for both backends — metric/PromQL
+ * sources surface native exemplars, trace sources generate them per bucket.
+ */
+// Both numbers are `.finite()` because both reach the renderer as SVG
+// coordinates: a non-finite timestamp collapses every affected exemplar into one
+// `@NaN` bucket and emits a NaN x-coordinate, and a single Infinity value makes
+// the 2σ spread NaN, silently switching the thinning rule off chart-wide. The
+// normalizers parse untrusted upstream bodies through this schema, so rejecting
+// here keeps both out of the render path.
+export const ExemplarSchema = z.object({
+ timestamp: z.number().finite(), // epoch ms, x-position on the chart
+ value: z.number().finite(), // exemplar's own value (metric Value / trace Duration in ms)
+ traceId: z.string(),
+ spanId: z.string().optional(),
+ // Matches the series the exemplar belongs to (LineData.displayName) so the
+ // overlay can attach markers to the right line. Undefined => applies to all.
+ groupKey: z.string().optional(),
+ attributes: z.record(z.string()).optional(),
+});
+
+export type Exemplar = z.infer;
+
export type DateRange = {
dateRange: [Date, Date];
dateRangeStartInclusive?: boolean; // default true
@@ -1669,6 +1702,9 @@ export const TeamClickHouseSettingsSchema = z.object({
metadataMaxRowsToRead: z.number().optional(),
parallelizeWhenPossible: z.boolean().optional(),
filterKeysFetchLimit: z.number().optional(),
+ // Target number of exemplar markers shown per chart (0 = unlimited). Not a
+ // ClickHouse setting, but lives in the same team-config bag for reuse.
+ maxExemplars: z.number().optional(),
});
/** Accepts null to unset (reset to default) a setting. */
@@ -1679,6 +1715,10 @@ export const TeamClickHouseSettingsUpdateSchema = z.object({
metadataMaxRowsToRead: z.number().nullish(),
parallelizeWhenPossible: z.boolean().nullish(),
filterKeysFetchLimit: z.number().nullish(),
+ // Bounded here, not just in the settings form: the form's min/max are
+ // client-side only, and a fractional or negative value reaches the chart's
+ // marker-budget arithmetic directly.
+ maxExemplars: z.number().int().min(0).max(1000).nullish(),
});
export type TeamClickHouseSettingsUpdate = z.infer<
typeof TeamClickHouseSettingsUpdateSchema