diff --git a/.changeset/add-search-histogram-severity-legend.md b/.changeset/add-search-histogram-severity-legend.md new file mode 100644 index 0000000000..9dce1dd233 --- /dev/null +++ b/.changeset/add-search-histogram-severity-legend.md @@ -0,0 +1,10 @@ +--- +'@hyperdx/app': minor +--- + +Add a legend below the search histogram showing each series' total across the +entire selected time range, so a breakdown like "how many errors in the last 45 +minutes" reads as one number instead of bars to sum by eye. Severity-like groups +are colored semantically and ordered most-severe-first; any other grouping uses +the chart's palette colors ordered by total. Clicking an item narrows the search +to that series. diff --git a/packages/app/src/ChartUtils.tsx b/packages/app/src/ChartUtils.tsx index d9e2ad8a70..f2f725de06 100644 --- a/packages/app/src/ChartUtils.tsx +++ b/packages/app/src/ChartUtils.tsx @@ -563,6 +563,23 @@ function setLineColors( }); } +/** + * Stack severity series in a consistent order: info at the bottom, then warn, + * then error on top. Series with no semantic color (any non-log-level grouping) + * all rank equally, so `sort` leaves them in the order the response produced. + */ +function sortLineDataByLogLevel(lineDataMap: { + [keyName: string]: LineDataWithOptionalColor; +}): LineDataWithOptionalColor[] { + const logLevelColorOrder = getLogLevelColorOrder(); + return Object.values(lineDataMap).sort((a, b) => { + return ( + logLevelColorOrder.findIndex(color => color === a.color) - + logLevelColorOrder.findIndex(color => color === b.color) + ); + }); +} + function firstGroupColumnIsLogLevel( source: TSource | undefined, groupColumns: ColumnMetaType[], @@ -736,13 +753,7 @@ export function formatResponseForTimeChart({ }); } - const logLevelColorOrder = getLogLevelColorOrder(); - const sortedLineData = Object.values(lineDataMap).sort((a, b) => { - return ( - logLevelColorOrder.findIndex(color => color === a.color) - - logLevelColorOrder.findIndex(color => color === b.color) - ); - }); + const sortedLineData = sortLineDataByLogLevel(lineDataMap); if (generateEmptyBuckets && granularity != null) { const generatedTsBuckets = timeBucketByGranularity( @@ -795,6 +806,88 @@ export function formatResponseForTimeChart({ }; } +/** One chart series collapsed to a single total over the whole date range. */ +export type SeriesTotal = { + /** The series key, identical to the chart's `Bar`/`Area` dataKey. */ + dataKey: string; + displayName: string; + /** The color the chart draws this series with. */ + color: string; + total: number; +}; + +/** + * Collapse a time-series response into one total per series, spanning the + * entire date range rather than a single bucket. + * + * This deliberately runs the same pipeline as `formatResponseForTimeChart` + * (identical series keys, the same stacking order, and `setLineColors` for + * colors) so a totals view can never disagree with the chart it summarizes — + * semantic colors when a group value looks like a log level, palette colors + * otherwise. Re-deriving any of that separately would drift the moment either + * side changed. + * + * Only the current period is summed: a previous-period comparison covers a + * different window, so folding it into a "total for this range" would be wrong. + */ +export function formatResponseForSeriesTotals({ + response, + source, +}: { + response: ResponseJSON>; + source?: TSource; +}): { + seriesTotals: SeriesTotal[]; + groupColumns: string[]; + isSingleValueColumn: boolean; +} { + const meta = response.meta; + if (meta == null) { + throw new Error('No meta data found in response'); + } + + const valueColumns = inferValueColumns(meta, new Set()) ?? []; + const groupColumns = inferGroupColumns(meta) ?? []; + + const tsBucketMap: Map> = new Map(); + const lineDataMap: { [keyName: string]: LineDataWithOptionalColor } = {}; + + addResponseToFormattedData({ + response, + lineDataMap, + tsBucketMap, + source, + isPreviousPeriod: false, + previousPeriodOffsetSeconds: 0, + }); + + const lineData = setLineColors(sortLineDataByLogLevel(lineDataMap)); + + const totalByDataKey = new Map(); + for (const bucket of tsBucketMap.values()) { + for (const line of lineData) { + const value = bucket[line.dataKey]; + if (typeof value === 'number' && Number.isFinite(value)) { + totalByDataKey.set( + line.dataKey, + (totalByDataKey.get(line.dataKey) ?? 0) + value, + ); + } + } + } + + return { + seriesTotals: lineData.map(line => ({ + dataKey: line.dataKey, + displayName: line.displayName || line.dataKey, + color: line.color, + total: totalByDataKey.get(line.dataKey) ?? 0, + })), + groupColumns: groupColumns.map(g => g.name), + isSingleValueColumn: valueColumns.length === 1, + }; +} + // Define a mapping from app AggFn to common-utils AggregateFunction const mapV1AggFnToV2 = (aggFn?: AggFn): AggFnV2 | undefined => { if (aggFn == null) { diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index e7ccda2b41..89009483fb 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -95,6 +95,7 @@ import { ErrorBoundary } from '@/components/Error/ErrorBoundary'; import { FavoriteButton } from '@/components/FavoriteButton'; import { InputControlled } from '@/components/InputControlled'; import OnboardingModal from '@/components/OnboardingModal'; +import SearchHistogramLegend from '@/components/SearchHistogramLegend'; import SearchWhereInput, { getStoredLanguage, } from '@/components/SearchInput/SearchWhereInput'; @@ -2433,6 +2434,14 @@ export function DBSearchPage() { /> )} + {!hasQueryError && ( + + )} )} + {!hasQueryError && ( + + )} )} {hasQueryError && queryError ? ( diff --git a/packages/app/src/__tests__/ChartUtils.test.ts b/packages/app/src/__tests__/ChartUtils.test.ts index a1ab4cbe8f..db2a464761 100644 --- a/packages/app/src/__tests__/ChartUtils.test.ts +++ b/packages/app/src/__tests__/ChartUtils.test.ts @@ -10,6 +10,7 @@ import { convertToTimeChartConfig, findNearestSeriesKey, formatResponseForCategoricalChart, + formatResponseForSeriesTotals, formatResponseForTimeChart, } from '@/ChartUtils'; import { COLORS } from '@/utils'; @@ -22,6 +23,12 @@ import { COLORS } from '@/utils'; const SEMANTIC_INFO_HEX = '#437eef'; const SEMANTIC_ERROR_HEX = '#ff725c'; +// A log source grouped by severity, which is what turns on semantic coloring. +const LOG_SOURCE = { + kind: SourceKind.Log, + severityTextExpression: 'SeverityText', +} as TSource; + describe('ChartUtils', () => { describe('formatResponseForTimeChart', () => { it('should throw an error if there is no timestamp column', () => { @@ -300,17 +307,12 @@ describe('ChartUtils', () => { ], }; - const source = { - kind: SourceKind.Log, - severityTextExpression: 'SeverityText', - } as TSource; - const actual = formatResponseForTimeChart({ currentPeriodResponse: res, dateRange: [new Date(), new Date()], granularity: '1 minute', generateEmptyBuckets: false, - source, + source: LOG_SOURCE, }); expect(actual.lineData).toEqual([ @@ -1171,4 +1173,209 @@ describe('ChartUtils', () => { expect(findNearestSeriesKey(seriesY, ['a', 'b'], 100, 30)).toBe('a'); }); }); + + describe('formatResponseForSeriesTotals', () => { + const SEVERITY_RESPONSE = { + data: [ + { + 'count()': '30', + SeverityText: 'info', + __hdx_time_bucket: '2025-11-26T12:23:00Z', + }, + { + 'count()': '4', + SeverityText: 'debug', + __hdx_time_bucket: '2025-11-26T12:23:00Z', + }, + { + 'count()': '2', + SeverityText: 'error', + __hdx_time_bucket: '2025-11-26T12:23:00Z', + }, + // Same series in a later bucket: totals must span the whole range. + { + 'count()': '20', + SeverityText: 'info', + __hdx_time_bucket: '2025-11-26T12:24:00Z', + }, + { + 'count()': '3', + SeverityText: 'error', + __hdx_time_bucket: '2025-11-26T12:24:00Z', + }, + ], + meta: [ + { name: 'count()', type: 'UInt64' }, + { name: 'SeverityText', type: 'LowCardinality(String)' }, + { name: '__hdx_time_bucket', type: 'DateTime' }, + ], + }; + + it('sums each series across every bucket in the range', () => { + const { seriesTotals } = formatResponseForSeriesTotals({ + response: SEVERITY_RESPONSE, + source: LOG_SOURCE, + }); + + expect( + seriesTotals.map(({ dataKey, total }) => ({ dataKey, total })), + ).toEqual([ + { dataKey: 'info', total: 50 }, + { dataKey: 'debug', total: 4 }, + { dataKey: 'error', total: 5 }, + ]); + }); + + it('keeps each raw severity value as its own series, like the chart does', () => { + const { seriesTotals } = formatResponseForSeriesTotals({ + response: SEVERITY_RESPONSE, + source: LOG_SOURCE, + }); + + // 'debug' is info-colored but is a separate stacked series, so it must + // not be folded into 'info'. + expect(seriesTotals.map(s => s.dataKey)).toEqual([ + 'info', + 'debug', + 'error', + ]); + expect(seriesTotals.find(s => s.dataKey === 'debug')?.color).toBe( + SEMANTIC_INFO_HEX, + ); + expect(seriesTotals.find(s => s.dataKey === 'error')?.color).toBe( + SEMANTIC_ERROR_HEX, + ); + }); + + it('assigns palette colors when the groups are not log levels', () => { + const response = { + data: [ + { + 'count()': '7', + ServiceName: 'checkout', + __hdx_time_bucket: '2025-11-26T12:23:00Z', + }, + { + 'count()': '9', + ServiceName: 'shipping', + __hdx_time_bucket: '2025-11-26T12:23:00Z', + }, + ], + meta: [ + { name: 'count()', type: 'UInt64' }, + { name: 'ServiceName', type: 'LowCardinality(String)' }, + { name: '__hdx_time_bucket', type: 'DateTime' }, + ], + }; + + const { seriesTotals, groupColumns } = formatResponseForSeriesTotals({ + response, + }); + + expect(groupColumns).toEqual(['ServiceName']); + expect(seriesTotals).toEqual([ + { + dataKey: 'checkout', + displayName: 'checkout', + color: COLORS[0], + total: 7, + }, + { + dataKey: 'shipping', + displayName: 'shipping', + color: COLORS[1], + total: 9, + }, + ]); + }); + + // The whole reason this shares `formatResponseForTimeChart`'s pipeline: a + // totals view must never disagree with the chart it summarizes. + it.each([ + ['a severity grouping', SEVERITY_RESPONSE, LOG_SOURCE], + [ + 'a non-severity grouping', + { + data: [ + { + 'count()': '7', + ServiceName: 'checkout', + __hdx_time_bucket: '2025-11-26T12:23:00Z', + }, + { + 'count()': '9', + ServiceName: 'shipping', + __hdx_time_bucket: '2025-11-26T12:23:00Z', + }, + ], + meta: [ + { name: 'count()', type: 'UInt64' }, + { name: 'ServiceName', type: 'LowCardinality(String)' }, + { name: '__hdx_time_bucket', type: 'DateTime' }, + ], + }, + undefined, + ], + ])( + 'matches the chart series keys, colors, and order for %s', + (_label, response, source) => { + const { seriesTotals } = formatResponseForSeriesTotals({ + response, + source, + }); + const { lineData } = formatResponseForTimeChart({ + currentPeriodResponse: response, + dateRange: [new Date(), new Date()], + granularity: '1 minute', + generateEmptyBuckets: false, + source, + }); + + expect( + seriesTotals.map(({ dataKey, displayName, color }) => ({ + dataKey, + displayName, + color, + })), + ).toEqual( + lineData.map(({ dataKey, displayName, color }) => ({ + dataKey, + displayName, + color, + })), + ); + }, + ); + + it('reports no group columns when the query is ungrouped', () => { + const { seriesTotals, groupColumns } = formatResponseForSeriesTotals({ + response: { + data: [ + { 'count()': '12', __hdx_time_bucket: '2025-11-26T12:23:00Z' }, + { 'count()': '8', __hdx_time_bucket: '2025-11-26T12:24:00Z' }, + ], + meta: [ + { name: 'count()', type: 'UInt64' }, + { name: '__hdx_time_bucket', type: 'DateTime' }, + ], + }, + }); + + expect(groupColumns).toEqual([]); + expect(seriesTotals).toEqual([ + { + dataKey: 'count()', + displayName: 'count()', + color: COLORS[0], + total: 20, + }, + ]); + }); + + it('throws when the response has no metadata', () => { + expect(() => + formatResponseForSeriesTotals({ response: { data: [] } }), + ).toThrow('No meta data found in response'); + }); + }); }); diff --git a/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx b/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx index 5c46ed7f18..c902bf4e3d 100644 --- a/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx +++ b/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx @@ -116,6 +116,8 @@ jest.mock('@/searchFilters', () => ({ jest.mock('@/hooks/useChartConfig', () => ({ useAliasMapFromChartConfig: () => ({ data: {} }), + // The histogram legend derives its totals from this query. + useQueriedChartConfig: jest.fn(() => ({ data: undefined, isLoading: false })), })); jest.mock('@/hooks/useExplainQuery', () => ({ diff --git a/packages/app/src/__tests__/DBSearchPageQueryKey.test.tsx b/packages/app/src/__tests__/DBSearchPageQueryKey.test.tsx index 210e970c26..e30a80be01 100644 --- a/packages/app/src/__tests__/DBSearchPageQueryKey.test.tsx +++ b/packages/app/src/__tests__/DBSearchPageQueryKey.test.tsx @@ -1,11 +1,14 @@ import React from 'react'; import objectHash from 'object-hash'; import { + BuilderChartConfigWithDateRange, ChartConfigWithDateRange, DisplayType, } from '@hyperdx/common-utils/dist/types'; +import { hashKey } from '@tanstack/react-query'; import { DBTimeChart } from '@/components/DBTimeChart'; +import SearchHistogramLegend from '@/components/SearchHistogramLegend'; import SearchTotalCountChart from '@/components/SearchTotalCountChart'; // Mock the API and hooks @@ -176,4 +179,87 @@ describe('DBSearchPage QueryKey Consistency', () => { const chartQueryKeyHash = objectHash(chartQueryKey); expect(searchQueryKeyHash).toBe(chartQueryKeyHash); }); + + // The severity legend re-aggregates the histogram's rows over the whole date + // range, so it must resolve from the histogram's cache entry rather than + // issuing a second ClickHouse query. React Query only dedupes when the keys + // hash identically, so assert against its own hashing function: a plain + // `toEqual` would pass even for keys that hash apart (e.g. an explicit + // `disableQueryChunking: undefined` vs `false`). + it('should use a queryKey that hashes identically to DBTimeChart for SearchHistogramLegend', () => { + const config: BuilderChartConfigWithDateRange = { + select: 'count()', + from: { databaseName: 'test', tableName: 'logs' }, + where: '', + timestampValueExpression: 'timestamp', + connection: 'test-connection', + displayType: DisplayType.StackedBar, + dateRange: [new Date('2024-01-01'), new Date('2024-01-02')], + }; + + const queryKeyPrefix = 'search'; + + // DBTimeChart also issues a (disabled) previous-period query whose key has + // no options element, so render each component in isolation and pick the + // main chunked query rather than indexing into a shared call list. + const renderAndGetPrimaryQueryKey = (element: React.ReactElement) => { + mockUseQueriedChartConfig.mockClear(); + renderWithMantine(element); + const keys = mockUseQueriedChartConfig.mock.calls + .map(call => call[1]?.queryKey) + .filter(key => key?.length === 4); + expect(keys).toHaveLength(1); + return keys[0]; + }; + + const chartQueryKey = renderAndGetPrimaryQueryKey( + , + ); + + const legendQueryKey = renderAndGetPrimaryQueryKey( + , + ); + + const totalCountQueryKey = renderAndGetPrimaryQueryKey( + , + ); + + expect(hashKey(legendQueryKey)).toBe(hashKey(chartQueryKey)); + expect(hashKey(totalCountQueryKey)).toBe(hashKey(chartQueryKey)); + }); + + it('should not pin disableQueryChunking in the legend queryKey when the histogram leaves it unset', () => { + const config: BuilderChartConfigWithDateRange = { + select: 'count()', + from: { databaseName: 'test', tableName: 'logs' }, + where: '', + timestampValueExpression: 'timestamp', + connection: 'test-connection', + displayType: DisplayType.StackedBar, + dateRange: [new Date('2024-01-01'), new Date('2024-01-02')], + }; + + renderWithMantine( + , + ); + + const legendQueryKey = mockUseQueriedChartConfig.mock.calls[0][1]?.queryKey; + + // `JSON.stringify` drops undefined values, so an unset flag must stay + // undefined rather than being normalized to `false` — otherwise the key + // hashes differently from the histogram's and the cache entry splits. + expect(legendQueryKey[3].disableQueryChunking).toBeUndefined(); + }); }); diff --git a/packages/app/src/components/SearchHistogramLegend.tsx b/packages/app/src/components/SearchHistogramLegend.tsx new file mode 100644 index 0000000000..9df767952c --- /dev/null +++ b/packages/app/src/components/SearchHistogramLegend.tsx @@ -0,0 +1,230 @@ +import { useMemo } from 'react'; +import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { Group, Popover, Text, UnstyledButton } from '@mantine/core'; + +import { formatResponseForSeriesTotals } from '@/ChartUtils'; +import { + decodeSeriesGroupFilters, + type SeriesGroupFilter, +} from '@/components/DBTimeChart'; +import { + type SearchHistogramQueryOptions, + useSearchHistogramQuery, +} from '@/hooks/useSearchHistogramQuery'; +import { useSource } from '@/source'; +import { getLogLevelClass } from '@/utils'; + +/** + * Inline items before overflowing into a "+N more" popover. The search legend + * spans the full width under the histogram, so it fits more than the chart + * tile's own legend (which is boxed into a dashboard tile), but it still has to + * stay one row so it doesn't push the results table down. + */ +const MAX_INLINE_ITEMS = 6; + +const LOG_LEVEL_RANK: Record = { error: 3, warn: 2, info: 1 }; + +type SeriesTotalItem = { + dataKey: string; + label: string; + color: string; + total: number; + /** Column/value pairs to filter on when this item is clicked. */ + groupFilters: SeriesGroupFilter[]; +}; + +/** + * How severe a series looks, based on its group values rather than on which + * column was grouped — the histogram groups by whatever the source designates, + * and the values may or may not be log levels. + */ +function getSeverityRank(groupFilters: SeriesGroupFilter[]): number { + let rank = 0; + for (const { value } of groupFilters) { + const logLevelClass = getLogLevelClass(value); + if (logLevelClass != null) { + rank = Math.max(rank, LOG_LEVEL_RANK[logLevelClass] ?? 0); + } + } + return rank; +} + +function useSearchSeriesTotals( + config: BuilderChartConfigWithDateRange, + queryKeyPrefix: string, + { + sourceId, + ...queryOptions + }: SearchHistogramQueryOptions & { sourceId?: string } = {}, +) { + // Resolves from the histogram's React Query cache entry, so the legend adds + // no query of its own. + const { data, isLoading } = useSearchHistogramQuery( + config, + queryKeyPrefix, + queryOptions, + ); + + // The same source the chart resolves, so severity/status groupings get the + // same semantic colors the bars are drawn with. + const { data: source } = useSource({ id: sourceId || config.source }); + + const items = useMemo(() => { + if (data?.meta == null || data.data == null) return []; + + let seriesTotals; + let groupColumns: string[]; + let isSingleValueColumn: boolean; + try { + ({ seriesTotals, groupColumns, isSingleValueColumn } = + formatResponseForSeriesTotals({ response: data, source })); + } catch (e) { + // Mirror the chart's handling of an unusable response shape: degrade to + // no legend rather than taking the search page down. + console.error(e); + return []; + } + + // Without a group-by there is a single series whose total is already shown + // as the result count above the histogram, so a legend would just repeat it. + if (groupColumns.length === 0) return []; + + return seriesTotals + .map(series => ({ + dataKey: series.dataKey, + label: series.displayName, + color: series.color, + total: series.total, + groupFilters: decodeSeriesGroupFilters({ + seriesKey: series.dataKey, + groupColumns, + isSingleValueColumn, + }), + })) + .filter(item => item.total > 0 && item.groupFilters.length > 0) + .sort( + (a, b) => + // Most important first. Severity-looking series lead with the most + // severe (matching how the chart's own legend lists the top of the + // stack first); anything else falls back to the biggest contributor, + // which also makes the "+N more" cutoff meaningful. + getSeverityRank(b.groupFilters) - getSeverityRank(a.groupFilters) || + b.total - a.total || + a.label.localeCompare(b.label), + ); + }, [data, source]); + + return { items, isLoading }; +} + +function LegendItem({ + item, + onFocusSeries, +}: { + item: SeriesTotalItem; + onFocusSeries?: (filters: SeriesGroupFilter[]) => void; +}) { + return ( + onFocusSeries?.(item.groupFilters)} + aria-label={`Filter by ${item.label}`} + title={`${item.label}: ${item.total.toLocaleString()}`} + > + +
+ + {item.label} + + + {item.total.toLocaleString()} + + + + ); +} + +/** + * Totals per histogram series across the whole selected time range, so a + * breakdown (e.g. "how many errors in the last 45 minutes") reads as one number + * instead of bars to sum by eye. Clicking an item narrows the search to it. + * + * Driven entirely by the groups the query returned: severity-like values are + * colored semantically and ordered most-severe-first, while any other grouping + * gets the chart's palette colors and is ordered by total. + */ +export default function SearchHistogramLegend({ + config, + queryKeyPrefix, + sourceId, + disableQueryChunking, + enableParallelQueries, + onFocusSeries, +}: { + config: BuilderChartConfigWithDateRange; + queryKeyPrefix: string; + sourceId?: string; + disableQueryChunking?: boolean; + enableParallelQueries?: boolean; + onFocusSeries?: (filters: SeriesGroupFilter[]) => void; +}) { + const { items } = useSearchSeriesTotals(config, queryKeyPrefix, { + sourceId, + disableQueryChunking, + enableParallelQueries, + }); + + if (items.length === 0) { + return null; + } + + const inlineItems = items.slice(0, MAX_INLINE_ITEMS); + const overflowItems = items.slice(MAX_INLINE_ITEMS); + + return ( + + {inlineItems.map(item => ( + + ))} + {overflowItems.length > 0 && ( + + + + + +{overflowItems.length} more + + + + + + {overflowItems.map(item => ( + + ))} + + + + )} + + ); +} diff --git a/packages/app/src/components/SearchTotalCountChart.tsx b/packages/app/src/components/SearchTotalCountChart.tsx index ec42f131fa..450d834175 100644 --- a/packages/app/src/components/SearchTotalCountChart.tsx +++ b/packages/app/src/components/SearchTotalCountChart.tsx @@ -1,69 +1,25 @@ import { useMemo } from 'react'; -import { - filterColumnMetaByType, - JSDataType, - ResponseJSON, -} from '@hyperdx/common-utils/dist/clickhouse'; import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import { Text } from '@mantine/core'; -import { keepPreviousData } from '@tanstack/react-query'; - -import api from '@/api'; -import { convertToTimeChartConfig } from '@/ChartUtils'; -import { useQueriedChartConfig } from '@/hooks/useChartConfig'; - -function inferCountColumn(meta: ResponseJSON['meta'] | undefined): string { - if (!meta) return 'count()'; - if (meta.find(col => col.name === 'count()')) { - return 'count()'; - } - // The column may be named differently, particularly when using Materialized Views. - return ( - filterColumnMetaByType(meta, [JSDataType.Number])?.[0].name ?? 'count()' - ); -} +import { + inferCountColumn, + type SearchHistogramQueryOptions, + useSearchHistogramQuery, +} from '@/hooks/useSearchHistogramQuery'; export function useSearchTotalCount( config: BuilderChartConfigWithDateRange, queryKeyPrefix: string, - { - disableQueryChunking, - enableParallelQueries, - }: { - disableQueryChunking?: boolean; - enableParallelQueries?: boolean; - } = {}, + options: SearchHistogramQueryOptions = {}, ) { - const { data: me, isLoading: isLoadingMe } = api.useMe(); - - // queriedConfig, queryKey, and enableQueryChunking match DBTimeChart so that react query can de-dupe these queries. - const queriedConfig = useMemo( - () => convertToTimeChartConfig(config), - [config], - ); + // Shares the histogram's React Query cache entry, so this adds no extra query. const { data: totalCountData, isLoading, isError, error, - } = useQueriedChartConfig(queriedConfig, { - queryKey: [ - queryKeyPrefix, - queriedConfig, - 'chunked', - { - disableQueryChunking, - enableParallelQueries, - parallelizeWhenPossible: me?.team?.parallelizeWhenPossible, - }, - ], - staleTime: 1000 * 60 * 5, - refetchOnWindowFocus: false, - placeholderData: keepPreviousData, // no need to flash loading state when in live tail - enableQueryChunking: true, - enabled: !isLoadingMe, - }); + } = useSearchHistogramQuery(config, queryKeyPrefix, options); const isTotalCountComplete = !!totalCountData?.isComplete; diff --git a/packages/app/src/components/__tests__/SearchHistogramLegend.test.tsx b/packages/app/src/components/__tests__/SearchHistogramLegend.test.tsx new file mode 100644 index 0000000000..93eb52c262 --- /dev/null +++ b/packages/app/src/components/__tests__/SearchHistogramLegend.test.tsx @@ -0,0 +1,308 @@ +import React from 'react'; +import { + BuilderChartConfigWithDateRange, + SourceKind, +} from '@hyperdx/common-utils/dist/types'; +import { screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; + +import SearchHistogramLegend from '@/components/SearchHistogramLegend'; +import { COLORS } from '@/utils'; + +jest.mock('@/hooks/useSearchHistogramQuery', () => ({ + ...jest.requireActual('@/hooks/useSearchHistogramQuery'), + useSearchHistogramQuery: jest.fn(), +})); + +jest.mock('@/source', () => ({ + ...jest.requireActual('@/source'), + useSource: jest.fn(), +})); + +// Grabbed off the mocked modules (rather than via `jest.mocked`) so the fixtures +// below can be the partial shapes these tests actually care about. +const { useSearchHistogramQuery: mockUseSearchHistogramQuery } = + jest.requireMock('@/hooks/useSearchHistogramQuery'); +const { useSource: mockUseSource } = jest.requireMock('@/source'); + +// Keep in sync with SEMANTIC_CHART_PALETTE in @/utils. +const SEMANTIC_INFO_HEX = '#437eef'; +const SEMANTIC_WARNING_HEX = '#efb118'; +const SEMANTIC_ERROR_HEX = '#ff725c'; + +const CONFIG: BuilderChartConfigWithDateRange = { + select: 'count()', + from: { databaseName: 'test', tableName: 'logs' }, + where: '', + timestampValueExpression: 'Timestamp', + connection: 'test-connection', + dateRange: [new Date('2024-01-01'), new Date('2024-01-02')], +}; + +// Only the fields the color/grouping logic reads; `useSource` is mocked, so +// this needs no cast to the full TSource union. +const LOG_SOURCE = { + kind: SourceKind.Log, + severityTextExpression: 'SeverityText', +}; + +type Row = { bucket: string; count: string | number; group: string }; + +function mockResponse( + rows: Row[], + { groupColumn = 'SeverityText' }: { groupColumn?: string } = {}, +) { + mockUseSearchHistogramQuery.mockReturnValue({ + data: { + meta: [ + { name: 'count()', type: 'UInt64' }, + { name: groupColumn, type: 'LowCardinality(String)' }, + { name: '__hdx_time_bucket', type: 'DateTime' }, + ], + data: rows.map(r => ({ + __hdx_time_bucket: r.bucket, + 'count()': r.count, + [groupColumn]: r.group, + })), + isComplete: true, + }, + isLoading: false, + }); +} + +function renderLegend( + onFocusSeries?: (filters: { column: string; value: string }[]) => void, +) { + return renderWithMantine( + , + ); +} + +/** The rendered item labels, in display order. */ +function legendLabels() { + return screen + .getAllByLabelText(/^Filter by /) + .map(el => el.getAttribute('aria-label')!.replace('Filter by ', '')); +} + +const BUCKET_A = '2024-01-01T00:00:00Z'; +const BUCKET_B = '2024-01-01T00:01:00Z'; + +describe('SearchHistogramLegend', () => { + beforeEach(() => { + mockUseSearchHistogramQuery.mockReset(); + mockUseSource.mockReset(); + mockUseSource.mockReturnValue({ data: LOG_SOURCE }); + }); + + it('sums each series across every bucket in the selected range', () => { + // The point of the legend: whole-range totals, not one bucket's values. + mockResponse([ + { bucket: BUCKET_A, count: '30', group: 'info' }, + { bucket: BUCKET_B, count: '20', group: 'info' }, + { bucket: BUCKET_A, count: '10', group: 'warn' }, + { bucket: BUCKET_B, count: '5', group: 'warn' }, + { bucket: BUCKET_B, count: '3', group: 'error' }, + ]); + + renderLegend(); + + expect(screen.getByLabelText('Filter by info')).toHaveTextContent('info50'); + expect(screen.getByLabelText('Filter by warn')).toHaveTextContent('warn15'); + expect(screen.getByLabelText('Filter by error')).toHaveTextContent( + 'error3', + ); + }); + + it('lists the most severe series first so error counts lead', () => { + mockResponse([ + { bucket: BUCKET_A, count: 50, group: 'info' }, + { bucket: BUCKET_A, count: 15, group: 'warn' }, + { bucket: BUCKET_A, count: 5, group: 'error' }, + ]); + + renderLegend(); + + expect(legendLabels()).toEqual(['error', 'warn', 'info']); + }); + + it('shows whatever severity values the query returned, without rolling them up', () => { + // 'debug' and 'info' are both info-colored but are distinct stacked series + // in the chart, so the legend must not merge them into one "Info" row. + mockResponse([ + { bucket: BUCKET_A, count: 30, group: 'info' }, + { bucket: BUCKET_A, count: 4, group: 'debug' }, + { bucket: BUCKET_A, count: 2, group: 'trace' }, + { bucket: BUCKET_A, count: 6, group: 'fatal' }, + ]); + + renderLegend(); + + expect(legendLabels()).toEqual(['fatal', 'info', 'debug', 'trace']); + expect(screen.getByLabelText('Filter by debug')).toHaveTextContent( + 'debug4', + ); + }); + + it('colors severity-like series semantically', () => { + mockResponse([ + { bucket: BUCKET_A, count: 5, group: 'info' }, + { bucket: BUCKET_A, count: 5, group: 'warn' }, + { bucket: BUCKET_A, count: 5, group: 'error' }, + ]); + + renderLegend(); + + const swatch = (label: string) => + screen.getByLabelText(`Filter by ${label}`).querySelector('div > div'); + expect(swatch('info')).toHaveStyle({ backgroundColor: SEMANTIC_INFO_HEX }); + expect(swatch('warn')).toHaveStyle({ + backgroundColor: SEMANTIC_WARNING_HEX, + }); + expect(swatch('error')).toHaveStyle({ + backgroundColor: SEMANTIC_ERROR_HEX, + }); + }); + + it('falls back to palette colors and biggest-first order for non-severity groups', () => { + mockUseSource.mockReturnValue({ data: undefined }); + mockResponse( + [ + { bucket: BUCKET_A, count: 7, group: 'checkout' }, + { bucket: BUCKET_A, count: 9, group: 'shipping' }, + ], + { groupColumn: 'ServiceName' }, + ); + + renderLegend(); + + expect(legendLabels()).toEqual(['shipping', 'checkout']); + // Palette colors are assigned in chart series order, not display order. + expect( + screen.getByLabelText('Filter by checkout').querySelector('div > div'), + ).toHaveStyle({ backgroundColor: COLORS[0] }); + expect( + screen.getByLabelText('Filter by shipping').querySelector('div > div'), + ).toHaveStyle({ backgroundColor: COLORS[1] }); + }); + + it('reports the clicked series as a column/value filter', async () => { + mockResponse([ + { bucket: BUCKET_A, count: 50, group: 'info' }, + { bucket: BUCKET_A, count: 5, group: 'error' }, + ]); + + const onFocusSeries = jest.fn(); + renderLegend(onFocusSeries); + + const user = userEvent.setup(); + await user.click(screen.getByLabelText('Filter by error')); + + expect(onFocusSeries).toHaveBeenCalledTimes(1); + expect(onFocusSeries).toHaveBeenCalledWith([ + { column: 'SeverityText', value: 'error' }, + ]); + }); + + it('moves series past the inline cap into a "+N more" popover', async () => { + mockUseSource.mockReturnValue({ data: undefined }); + mockResponse( + Array.from({ length: 9 }, (_, i) => ({ + bucket: BUCKET_A, + count: 100 - i, + group: `service-${i}`, + })), + { groupColumn: 'ServiceName' }, + ); + + renderLegend(); + + // Six inline, the rest behind the overflow toggle. + expect(legendLabels()).toEqual([ + 'service-0', + 'service-1', + 'service-2', + 'service-3', + 'service-4', + 'service-5', + ]); + expect(screen.getByText('+3 more')).toBeInTheDocument(); + + const user = userEvent.setup(); + await user.click(screen.getByLabelText('Show remaining series')); + + expect( + await screen.findByLabelText('Filter by service-8'), + ).toHaveTextContent('service-892'); + }); + + it('renders nothing for an ungrouped query, whose total is already shown above', () => { + mockUseSearchHistogramQuery.mockReturnValue({ + data: { + meta: [ + { name: 'count()', type: 'UInt64' }, + { name: '__hdx_time_bucket', type: 'DateTime' }, + ], + data: [{ __hdx_time_bucket: BUCKET_A, 'count()': '70' }], + isComplete: true, + }, + isLoading: false, + }); + + renderLegend(); + + expect( + screen.queryByTestId('search-histogram-legend'), + ).not.toBeInTheDocument(); + }); + + it('renders nothing when the range has no rows', () => { + mockResponse([]); + + renderLegend(); + + expect( + screen.queryByTestId('search-histogram-legend'), + ).not.toBeInTheDocument(); + }); + + it('renders nothing while the shared histogram query is still loading', () => { + mockUseSearchHistogramQuery.mockReturnValue({ + data: undefined, + isLoading: true, + }); + + renderLegend(); + + expect( + screen.queryByTestId('search-histogram-legend'), + ).not.toBeInTheDocument(); + }); + + it('renders nothing rather than throwing on an unusable response shape', () => { + const consoleError = jest + .spyOn(console, 'error') + .mockImplementation(() => undefined); + // No timestamp column: the chart formatter throws on this. + mockUseSearchHistogramQuery.mockReturnValue({ + data: { + meta: [{ name: 'count()', type: 'UInt64' }], + data: [{ 'count()': '1' }], + isComplete: true, + }, + isLoading: false, + }); + + renderLegend(); + + expect( + screen.queryByTestId('search-histogram-legend'), + ).not.toBeInTheDocument(); + expect(consoleError).toHaveBeenCalled(); + consoleError.mockRestore(); + }); +}); diff --git a/packages/app/src/hooks/useSearchHistogramQuery.ts b/packages/app/src/hooks/useSearchHistogramQuery.ts new file mode 100644 index 0000000000..b9d940f24a --- /dev/null +++ b/packages/app/src/hooks/useSearchHistogramQuery.ts @@ -0,0 +1,80 @@ +import { useMemo } from 'react'; +import { + filterColumnMetaByType, + JSDataType, + ResponseJSON, +} from '@hyperdx/common-utils/dist/clickhouse'; +import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; +import { keepPreviousData } from '@tanstack/react-query'; + +import api from '@/api'; +import { convertToTimeChartConfig } from '@/ChartUtils'; +import { useQueriedChartConfig } from '@/hooks/useChartConfig'; + +export type SearchHistogramQueryOptions = { + disableQueryChunking?: boolean; + enableParallelQueries?: boolean; +}; + +/** + * The single source of truth for the search page's histogram query. + * + * The histogram (DBTimeChart), the total result count, and the severity legend + * all need the same grouped-by-severity time series. Rather than each issuing + * its own query, they all resolve from one React Query cache entry — which only + * works while their query keys hash identically. Building that key in more than + * one place has already proven easy to get subtly wrong (an explicit + * `disableQueryChunking: false` hashes differently from an absent one, because + * `JSON.stringify` drops undefined object values), so every search-page + * consumer must go through this hook instead of assembling a key by hand. + * + * The key/option shape here must stay in sync with DBTimeChart's. That + * invariant is covered by `__tests__/DBSearchPageQueryKey.test.tsx`. + */ +export function useSearchHistogramQuery( + config: BuilderChartConfigWithDateRange, + queryKeyPrefix: string, + { + disableQueryChunking, + enableParallelQueries, + }: SearchHistogramQueryOptions = {}, +) { + const { data: me, isLoading: isLoadingMe } = api.useMe(); + + const queriedConfig = useMemo( + () => convertToTimeChartConfig(config), + [config], + ); + + return useQueriedChartConfig(queriedConfig, { + queryKey: [ + queryKeyPrefix, + queriedConfig, + 'chunked', + { + disableQueryChunking, + enableParallelQueries, + parallelizeWhenPossible: me?.team?.parallelizeWhenPossible, + }, + ], + staleTime: 1000 * 60 * 5, + refetchOnWindowFocus: false, + placeholderData: keepPreviousData, // no need to flash loading state when in live tail + enableQueryChunking: true, + enabled: !isLoadingMe, + }); +} + +export function inferCountColumn( + meta: ResponseJSON['meta'] | undefined, +): string { + if (!meta) return 'count()'; + if (meta.find(col => col.name === 'count()')) { + return 'count()'; + } + + // The column may be named differently, particularly when using Materialized Views. + return ( + filterColumnMetaByType(meta, [JSDataType.Number])?.[0]?.name ?? 'count()' + ); +}