feat(app): exemplar overlay for metric and PromQL time charts

.changeset/exemplar-overlay.md

@@ -0,0 +1,25 @@
+---
+'@hyperdx/common-utils': minor
+'@hyperdx/app': minor
+---
+
+feat: exemplar overlay for metric and PromQL time charts
+
+Time charts on metric and PromQL sources can overlay exemplars — individual
+trace-linked data points — via the "Exemplars" toggle in the chart editor.
+Hovering a marker shows the exemplar's own value and time plus trace metadata
+from a configurable trace source, with a button to open the trace.
+
+Off by default for the whole deployment behind `NEXT_PUBLIC_ENABLE_EXEMPLARS`,
+and per-chart behind `enableExemplars`.
+
+Markers are sampled the way Grafana samples them: bucketed at the chart's
+granularity, keeping the slowest trace in each bucket plus any further trace more
+than 2σ below it. The overlay shows the shape of the latency distribution rather
+than tracing the top of the chart.
+
+A marker sits at the trace's own measurement, so it is only shown where that is
+honest: a single non-ratio histogram series with no group by, aggregated in a way
+that leaves the axis on the observation scale, and for PromQL an expression that
+plots a duration. Markers outside the rendered window, or below a fitted y-axis
+floor, are dropped rather than moved — and the count is surfaced on the chart.

packages/app/package.json

@@ -14,7 +14,7 @@
     "build:clickhouse": "NEXT_PUBLIC_THEME=clickstack NEXT_PUBLIC_IS_LOCAL_MODE=true NEXT_PUBLIC_CLICKHOUSE_BUILD=true next build --webpack && node scripts/prepare-clickhouse-build-export.js",
     "run:clickhouse": "test -d out && npx rimraf tmp && mkdir tmp && cp -r out tmp/clickstack && echo 'visit http://localhost:3000/clickstack to start' && npx serve tmp -l 3000 || echo 'run build:clickhouse first'",
     "start": "next start",
-    "lint": "npx eslint . --ext .ts,.tsx --max-warnings 663",
+    "lint": "npx eslint . --ext .ts,.tsx --max-warnings 668",
     "lint:fix": "npx eslint . --ext .ts,.tsx --fix",
     "lint:styles": "stylelint **/*/*.{css,scss}",
     "ci:lint": "yarn lint && yarn tsc --noEmit && yarn lint:styles --quiet",

packages/app/src/HDXMultiSeriesTimeChart/MemoChart.tsx

@@ -15,19 +15,21 @@ import {
   CartesianGrid,
   Legend,
   ReferenceArea,
+  ReferenceDot,
   ReferenceLine,
   ResponsiveContainer,
   Tooltip,
   XAxis,
   YAxis,
 } from 'recharts';
-import { DisplayType } from '@hyperdx/common-utils/dist/types';
+import { DisplayType, Exemplar } from '@hyperdx/common-utils/dist/types';
 
 import { useChartSyncId } from '@/chartSync';
 import { findNearestSeriesKey, LineData } from '@/ChartUtils';
 import { ChartAnnotation } from '@/components/charts/chartAnnotations';
 import { ChartOverlayControls } from '@/components/charts/ChartOverlayControls';
 import { toViewportPoint } from '@/components/charts/ChartTooltip';
+import { ExemplarDot } from '@/components/Exemplars';
 import type { NumberFormat } from '@/types';
 import { useFormatTime } from '@/useFormatTime';
 import { COLORS, formatNumber } from '@/utils';
@@ -51,6 +53,7 @@ import {
   Y_AXIS_WIDTH,
 } from './constants';
 import { useChartScales } from './useChartScales';
+import { useExemplarMarkers } from './useExemplarMarkers';
 
 // Debounce (ms) for the chart's ResponsiveContainer resize observer. Without
 // it the observer fires on every frame, and a resize → re-render → resize
@@ -108,6 +111,15 @@ export const MemoChart = memo(function MemoChart({
   granularity,
   dateRangeEndInclusive = true,
   fitYAxisToData = false,
+  exemplars,
+  maxExemplars = 12,
+  onExemplarHover,
+  onExemplarHoverEnd,
+  onExemplarSelect,
+  pinnedExemplarKey = null,
+  onExemplarPinEnd,
+  onActiveExemplarMoved,
+  onExemplarsDropped,
 }: {
   // Matches what useChartScales narrows to, so the hook's stricter type is
   // actually checked at this boundary rather than satisfied by `any`.
@@ -145,9 +157,31 @@ export const MemoChart = memo(function MemoChart({
    * (with padding) instead of zero.
    **/
   fitYAxisToData?: boolean;
+  /** Exemplar markers to overlay on the chart (linked to traces). */
+  exemplars?: Exemplar[];
+  /** Target number of exemplar markers to show (0 = unlimited). */
+  maxExemplars?: number;
+  /** Invoked when the cursor enters an exemplar marker, with its pixel coords. */
+  onExemplarHover?: (exemplar: Exemplar, cx: number, cy: number) => void;
+  /** Invoked when the cursor leaves an exemplar marker. */
+  onExemplarHoverEnd?: () => void;
+  /** Invoked when an exemplar marker is clicked, with its pixel coords. */
+  onExemplarSelect?: (exemplar: Exemplar, cx: number, cy: number) => void;
+  /**
+   * Key of the exemplar whose card is pinned open, or null. A key rather than a
+   * boolean so the chart can tell when that marker stops being rendered — see
+   * the reset effect below. A pin also suppresses the series tooltip.
+   */
+  pinnedExemplarKey?: string | null;
+  /** Invoked when the pinned marker is no longer in the rendered set. */
+  onExemplarPinEnd?: () => void;
+  /** See useExemplarMarkers: re-anchors the open card when its marker moves. */
+  onActiveExemplarMoved?: (cx: number, cy: number) => void;
+  /** How many markers the render-layer clamps dropped; see useExemplarMarkers. */
+  onExemplarsDropped?: (count: number) => void;
 }) {
-  const _id = useId();
-  const id = _id.replace(/:/g, '');
+  const rawId = useId();
+  const id = rawId.replace(/:/g, '');
 
   // recharts sync group, scoped via context (see chartSync).
   const syncId = useChartSyncId();
@@ -248,18 +282,21 @@ export const MemoChart = memo(function MemoChart({
     captureActivePointY,
   ]);
 
-  // Axis domains and annotation elements — see useChartScales.
-  const { yAxisDomain, xAxisDomain, annotationElements } = useChartScales({
-    annotations,
-    dateRange,
-    granularity,
-    dateRangeEndInclusive,
-    displayType,
-    fitYAxisToData,
-    graphResults,
-    lineData,
-    selectedSeriesNames,
-  });
+  // Axis domains, the exemplar clamp range, and annotation elements — see
+  // useChartScales.
+  const { yAxisDomain, exemplarYBounds, xAxisDomain, annotationElements } =
+    useChartScales({
+      annotations,
+      dateRange,
+      granularity,
+      dateRangeEndInclusive,
+      displayType,
+      fitYAxisToData,
+      graphResults,
+      lineData,
+      selectedSeriesNames,
+      hasExemplars: !!exemplars?.length,
+    });
 
   const [containerWidth, setContainerWidth] = useState(0);
 
@@ -415,6 +452,31 @@ export const MemoChart = memo(function MemoChart({
     return map;
   }, [lineData]);
 
+  // Exemplar marker layer — see useExemplarMarkers.
+  const {
+    activeExemplarKey,
+    exemplarPoints,
+    isExemplarHovered,
+    handleExemplarHoverStart,
+    handleExemplarHoverEnd,
+    handleExemplarSelect,
+  } = useExemplarMarkers({
+    exemplars,
+    maxExemplars,
+    granularity,
+    pinnedExemplarKey,
+    xAxisDomain,
+    exemplarYBounds,
+    onExemplarHover,
+    onExemplarHoverEnd,
+    onExemplarSelect,
+    onExemplarPinEnd,
+    onActiveExemplarMoved,
+    onExemplarsDropped,
+    suppressNextClickRef,
+    brushOriginRef: mouseDownPosRef,
+  });
+
   return (
     <div
       ref={containerRef}
@@ -655,23 +717,48 @@ export const MemoChart = memo(function MemoChart({
               Hidden once a point is clicked, where the pinned tooltip takes over.
               Portaled to body so HDXLineChartTooltip can self-position (see its
               docblock) and escape the chart's bounds near an edge. */}
-          {isClickActive == null && (
-            <Tooltip
-              content={
-                <HDXLineChartTooltip
-                  numberFormat={fallbackNumberFormat}
-                  numberFormatByKey={tooltipNumberFormatsByKey}
-                  lineDataMap={lineDataMap}
-                  previousPeriodOffsetSeconds={previousPeriodOffsetSeconds}
-                  activePointYByKeyRef={activePointYByKeyRef}
-                  containerRef={containerRef}
+          {isClickActive == null &&
+            !isExemplarHovered &&
+            pinnedExemplarKey == null && (
+              <Tooltip
+                content={
+                  <HDXLineChartTooltip
+                    numberFormat={fallbackNumberFormat}
+                    numberFormatByKey={tooltipNumberFormatsByKey}
+                    lineDataMap={lineDataMap}
+                    previousPeriodOffsetSeconds={previousPeriodOffsetSeconds}
+                    activePointYByKeyRef={activePointYByKeyRef}
+                    containerRef={containerRef}
+                  />
+                }
+                portal={typeof document !== 'undefined' ? document.body : null}
+              />
+            )}
+          {referenceLines}
+          {annotationElements}
+          {exemplarPoints.map(p => (
+            <ReferenceDot
+              key={p.key}
+              // Already placed inside the x-domain by useExemplarMarkers, which
+              // also drops markers that belong to a different window.
+              x={p.x}
+              // Already placed inside the y-range by useExemplarMarkers.
+              y={p.y}
+              // Stated rather than assumed: both clamps exist because the default
+              // is "discard", and recharts 3 landed here recently.
+              ifOverflow="discard"
+              shape={
+                <ExemplarDot
+                  exemplar={p.exemplar}
+                  onHoverStart={handleExemplarHoverStart}
+                  onHoverEnd={handleExemplarHoverEnd}
+                  onSelect={handleExemplarSelect}
+                  isActive={p.key === activeExemplarKey}
+                  onPositionChange={onActiveExemplarMoved}
                 />
               }
-              portal={typeof document !== 'undefined' ? document.body : null}
             />
-          )}
-          {referenceLines}
-          {annotationElements}
+          ))}
           {highlightStart && highlightEnd ? (
             <ReferenceArea
               // yAxisId="1"

packages/app/src/HDXMultiSeriesTimeChart/__tests__/useChartScales.test.ts

@@ -39,6 +39,7 @@ const baseArgs = {
   ] as Record<string, unknown>[],
   lineData,
   selectedSeriesNames: undefined as Set<string> | undefined,
+  hasExemplars: false,
 };
 
 const scales = (overrides: Partial<typeof baseArgs> = {}) =>
@@ -110,6 +111,43 @@ describe('useChartScales y-domain', () => {
   });
 });
 
+describe('useChartScales exemplar clamp', () => {
+  // With no selection and no fit the y-domain upper bound is 'auto', so the
+  // clamp falls back to the visible series max — which is the whole point: an
+  // outlier marker pins to the top of the series range instead of stretching the
+  // axis and flattening every line.
+  it('bounds markers by the visible series max', () => {
+    expect(scales({ hasExemplars: true }).exemplarYBounds).toEqual({
+      min: 0,
+      max: 40,
+    });
+  });
+
+  it('follows the legend selection rather than every series', () => {
+    // A selection gives the axis numeric bounds, so the clamp takes those
+    // directly ([9, 31]) instead of falling back to the series max. Either way
+    // it tracks what is on screen — with B shown too it would be [8.5, 41.5].
+    expect(
+      scales({ hasExemplars: true, selectedSeriesNames: new Set(['A']) })
+        .exemplarYBounds,
+    ).toEqual({ min: 9, max: 31 });
+  });
+
+  it('skips the O(rows x series) scan when no marker can draw', () => {
+    // Every time chart in the app pays for this pass otherwise, including
+    // deployments running with the overlay switched off entirely.
+    expect(scales({ hasExemplars: false }).exemplarYBounds.max).toBe(0);
+  });
+
+  it('takes a numeric upper bound from the axis when there is one', () => {
+    // Fitting to data gives a real number, and the marker should respect the
+    // axis the chart actually drew rather than the raw series max.
+    expect(
+      scales({ hasExemplars: true, fitYAxisToData: true }).exemplarYBounds,
+    ).toEqual({ min: 8.5, max: 41.5 });
+  });
+});
+
 describe('useChartScales x-domain', () => {
   it('spans the requested range in seconds', () => {
     const [start, end] = scales().xAxisDomain;