diff --git a/.changeset/quiet-lions-repeat.md b/.changeset/quiet-lions-repeat.md new file mode 100644 index 0000000000..93c6b47726 --- /dev/null +++ b/.changeset/quiet-lions-repeat.md @@ -0,0 +1,11 @@ +--- +'@hyperdx/common-utils': minor +--- + +Fix legacy `filters` migration to merge with the existing `where` clause instead of replacing it, so filters are never dropped when both are present. + +- Add `mergeFilterStateIntoWhereClause` to AND existing `where` text with migrated filters (verbatim preservation, OR-wrapping of the residual, SQL-safe when needed). +- Fix invalid/incomplete Lucene clauses (e.g. mid-edit `service:`) so `replaceLuceneFacetClauses` now emits the new clause instead of no-oping, and `replaceFilterClauses` reports them via `getWhereParseError`. +- Support Lucene `NOT` negation (`NOT field:"v"`, `field:"v" AND NOT other:"w"`) in facet parsing so the sidebar shows an excluded (indeterminate) state instead of a checked one. +- Add `getUnrepresentableWhereReason` so cross-field `OR` queries can be surfaced to the user rather than silently misrepresented as AND. +- Add `emitLanguage` to `replaceFilterClauses` so facet clauses can be rewritten across query languages (Lucene ↔ SQL) while preserving non-facet text — used by the UI on language switch. diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index ecd4638484..a73ce37a93 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -116,7 +116,15 @@ import { useSavedSearch, useUpdateSavedSearch, } from '@/savedSearch'; -import { useSearchPageFilterState } from '@/searchFilters'; +import { + getUnrepresentableWhereReason, + getWhereParseError, + mergeFiltersIntoWhereClause, + replaceFiltersInWhereClause, + translateWhereClauseInQuery, + useSearchPageFilterState, + whereToFilters, +} from '@/searchFilters'; import { getEventBody, useSource, useSources } from '@/source'; import { useAppTheme, useBrandDisplayName } from '@/theme/ThemeProvider'; import { @@ -1098,7 +1106,7 @@ export function DBSearchPage() { [sources, lastSelectedSourceId], ); - const { control, setValue, reset, handleSubmit, formState } = + const { control, setValue, reset, getValues, handleSubmit, formState } = useForm({ values: { select: searchedConfig.select || '', @@ -1263,18 +1271,15 @@ export function DBSearchPage() { const onSubmit = useCallback(() => { onSearch(displayedTimeInputValue); - handleSubmit( - ({ select, where, whereLanguage, source, filters, orderBy }) => { - setSearchedConfig({ - select, - where, - whereLanguage, - source, - filters, - orderBy, - }); - }, - )(); + handleSubmit(({ select, where, whereLanguage, source, orderBy }) => { + setSearchedConfig({ + select, + where, + whereLanguage, + source, + orderBy, + }); + })(); setPatternColumn(draftPatternColumn || null); // clear query errors setQueryErrors({}); @@ -1289,13 +1294,6 @@ export function DBSearchPage() { ]); const debouncedSubmit = useDebouncedCallback(onSubmit, 1000); - const handleSetFilters = useCallback( - (filters: Filter[]) => { - setValue('filters', filters); - debouncedSubmit(); - }, - [debouncedSubmit, setValue], - ); // Top-level column names for the active source, used to quote // filter keys that contain special characters. @@ -1343,14 +1341,131 @@ export function DBSearchPage() { const { dateTimeColumns, onResolvedColumnsChange } = useResolvedDateTimeColumns(inputSourceColumns); - const filters = useWatch({ name: 'filters', control }); + // The `where` clause is the canonical representation of the query. The + // sidebar's FilterState is derived from it (via `whereToFilters`) and sidebar + // toggles rewrite it in place (via `replaceFiltersInWhereClause`), so the + // filter sidebar and the query text can never drift apart. + const inputWhere = useWatch({ name: 'where', control }); + const inputWhereLanguage = useWatch({ name: 'whereLanguage', control }); + + // The `where` text is the canonical representation, but while the user is + // mid-typing it can be momentarily unparseable (e.g. `service:`). Keep the + // last valid derived query so the sidebar doesn't wipe, and surface the + // parse error separately so the sidebar can explain what's happening. + const lastGoodSearchQueryRef = useRef([]); + const searchQuery = useMemo(() => { + const language = inputWhereLanguage ?? 'lucene'; + const parseError = getWhereParseError(inputWhere, language); + if (parseError) return lastGoodSearchQueryRef.current; + const query = whereToFilters( + inputWhere, + language, + knownColumns, + dateTimeColumns, + ); + lastGoodSearchQueryRef.current = query; + return query; + }, [inputWhere, inputWhereLanguage, knownColumns, dateTimeColumns]); + + const whereParseError = useMemo( + () => getWhereParseError(inputWhere, inputWhereLanguage ?? 'lucene'), + [inputWhere, inputWhereLanguage], + ); + + const whereUnrepresentableReason = useMemo( + () => + getUnrepresentableWhereReason(inputWhere, inputWhereLanguage ?? 'lucene'), + [inputWhere, inputWhereLanguage], + ); + + // Language switches happen through SearchWhereInput; translate the live + // `where` text so facet clauses survive (the free-text/facet rewrite keeps + // non-facet content verbatim). Skip when there is nothing to translate. + const handleWhereLanguageChange = useCallback( + (language: 'sql' | 'lucene') => { + const previousLanguage = inputWhereLanguage ?? 'lucene'; + if (language === previousLanguage) return; + const translatedWhere = translateWhereClauseInQuery( + inputWhere, + previousLanguage, + language, + knownColumns, + dateTimeColumns, + ); + if (translatedWhere !== inputWhere) { + setValue('where', translatedWhere, { shouldDirty: true }); + } + }, + [inputWhere, inputWhereLanguage, knownColumns, dateTimeColumns, setValue], + ); + + // Sidebar filter mutations arrive as the canonical SQL `Filter[]` the hook + // emits; rewrite the facet clauses of the live `where` text to match and + // drop the separate `filters` param entirely. + // + // Read the current form values with `getValues()` (a non-reactive read) at + // call time instead of closing over the watched `where`, so this callback + // keeps a stable identity across keystrokes. A stable `onFilterChange` keeps + // the sidebar hook's eight mutators stable too, which is what lets + // `memo(DBSearchPageFiltersComponent)` skip re-rendering while typing. + const handleSetFilters = useCallback( + (filters: Filter[]) => { + const newWhere = replaceFiltersInWhereClause( + getValues('where') ?? '', + getValues('whereLanguage') ?? 'lucene', + filters, + knownColumns, + dateTimeColumns, + ); + setValue('where', newWhere); + debouncedSubmit(); + }, + [debouncedSubmit, setValue, getValues, knownColumns, dateTimeColumns], + ); + const searchFilters = useSearchPageFilterState({ - searchQuery: filters ?? undefined, + searchQuery, onFilterChange: handleSetFilters, dateTimeColumns, knownColumns, }); + // One-time migration: legacy `filters` params (URL, saved search) move into + // the `where` clause, now the canonical representation. Gated on the source's + // columns being known so date columns and special-character keys emit + // correctly, then `filters` is cleared so the page stops persisting it. + useEffect(() => { + const { filters: legacyFilters, where, whereLanguage } = searchedConfig; + if (!legacyFilters?.length || !inputSourceColumns) return; + try { + const migratedWhere = mergeFiltersIntoWhereClause( + where ?? '', + (whereLanguage as 'sql' | 'lucene') ?? 'lucene', + legacyFilters, + knownColumns, + dateTimeColumns, + ); + if (migratedWhere !== (where ?? '')) { + setSearchedConfig({ where: migratedWhere, filters: [] }); + } else if (where) { + // Filters were already represented in the where text — just stop + // persisting the redundant param. + setSearchedConfig({ filters: [] }); + } + } catch (e) { + console.error('Failed to migrate legacy filters into where clause', e); + } + // Runs when a legacy filters-bearing config is present; the emit depends on + // columns loading so re-check when they arrive. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [ + searchedConfig.filters, + searchedConfig.where, + searchedConfig.whereLanguage, + inputSourceColumns, + setSearchedConfig, + ]); + useEffect(() => { // If the user changes the source dropdown, reset the select and orderby fields // to match the new source selected @@ -1493,8 +1608,6 @@ export function DBSearchPage() { : null; return { hasQueryError, queryError }; }, [_queryErrors]); - const inputWhere = useWatch({ name: 'where', control }); - const inputWhereLanguage = useWatch({ name: 'whereLanguage', control }); // query suggestion for 'where' if error const whereSuggestions = useSqlSuggestions({ input: inputWhere, @@ -1712,7 +1825,6 @@ export function DBSearchPage() { } else { qParams.append('select', searchedConfig.select || ''); qParams.append('where', where || searchedConfig.where || ''); - qParams.append('filters', JSON.stringify(searchedConfig.filters ?? [])); qParams.append('source', searchedSource?.id || ''); } @@ -1720,7 +1832,6 @@ export function DBSearchPage() { }, [ interval, - searchedConfig.filters, searchedConfig.select, searchedConfig.where, searchedSource?.id, @@ -2314,6 +2425,7 @@ export function DBSearchPage() { control={control} name="where" onSubmit={onSubmit} + onLanguageChange={handleWhereLanguageChange} sqlQueryHistoryType={QUERY_LOCAL_STORAGE.SEARCH_SQL} luceneQueryHistoryType={QUERY_LOCAL_STORAGE.SEARCH_LUCENE} enableHotkey @@ -2431,6 +2543,8 @@ export function DBSearchPage() { onColumnToggle={toggleColumn} displayedColumns={displayedColumns} onCollapse={() => setIsFilterSidebarCollapsed(true)} + whereParseError={whereParseError} + whereUnrepresentableReason={whereUnrepresentableReason} {...searchFilters} /> diff --git a/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx b/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx index d3f10d02a5..73639e295d 100644 --- a/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx +++ b/packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx @@ -118,6 +118,12 @@ jest.mock('@/searchFilters', () => ({ setFilterValue: jest.fn(), clearAllFilters: jest.fn(), }), + whereToFilters: () => [], + replaceFiltersInWhereClause: (where: string) => where, + mergeFiltersIntoWhereClause: (where: string) => where, + translateWhereClauseInQuery: (where: string) => where, + getWhereParseError: () => null, + getUnrepresentableWhereReason: () => null, })); jest.mock('@/hooks/useChartConfig', () => ({ diff --git a/packages/app/src/__tests__/SessionSidePanel.test.tsx b/packages/app/src/__tests__/SessionSidePanel.test.tsx index 5bf4b5e467..078b19e2ac 100644 --- a/packages/app/src/__tests__/SessionSidePanel.test.tsx +++ b/packages/app/src/__tests__/SessionSidePanel.test.tsx @@ -40,7 +40,7 @@ jest.mock('nuqs', () => { const noop = () => {}; return { ...actual, - // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useQueryState: (key: string, parser?: { defaultValue?: unknown }) => key === 'sessionPanelEvent' ? [mockNuqs.sessionPanelEvent, mockNuqs.setSessionPanelEvent] diff --git a/packages/app/src/components/DBSearchPageFilters.tsx b/packages/app/src/components/DBSearchPageFilters.tsx index 67a154aae7..c565c6ee86 100644 --- a/packages/app/src/components/DBSearchPageFilters.tsx +++ b/packages/app/src/components/DBSearchPageFilters.tsx @@ -12,6 +12,7 @@ import { import { Accordion, ActionIcon, + Alert, Box, Button, Center, @@ -33,6 +34,7 @@ import { } from '@mantine/core'; import { notifications } from '@mantine/notifications'; import { + IconAlertCircle, IconArrowBarToLeft, IconChartBar, IconChartBarOff, @@ -1072,6 +1074,8 @@ const DBSearchPageFiltersComponent = ({ onColumnToggle, displayedColumns, onCollapse, + whereParseError, + whereUnrepresentableReason, }: { analysisMode: 'results' | 'delta' | 'pattern'; setAnalysisMode: (mode: 'results' | 'delta' | 'pattern') => void; @@ -1085,6 +1089,8 @@ const DBSearchPageFiltersComponent = ({ onColumnToggle?: (column: string) => void; displayedColumns?: string[]; onCollapse?: () => void; + whereParseError?: string | null; + whereUnrepresentableReason?: string | null; } & FilterStateHook) => { const setFilterValue = useCallback( ( @@ -1662,6 +1668,35 @@ const DBSearchPageFiltersComponent = ({ )} + {whereParseError && ( + } + > + + Finish the query text above, or click a value below to replace + the incomplete text. + + + )} + {!whereParseError && whereUnrepresentableReason && ( + } + > + + {whereUnrepresentableReason} + + + )} diff --git a/packages/app/src/components/__tests__/DBRowSidePanel.missingSource.test.tsx b/packages/app/src/components/__tests__/DBRowSidePanel.missingSource.test.tsx index 6174961a7f..53d7ae4793 100644 --- a/packages/app/src/components/__tests__/DBRowSidePanel.missingSource.test.tsx +++ b/packages/app/src/components/__tests__/DBRowSidePanel.missingSource.test.tsx @@ -26,7 +26,7 @@ jest.mock('nuqs', () => { const actual = jest.requireActual('nuqs'); return { ...actual, - // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useQueryState: (key: string, parser?: { defaultValue?: unknown }) => { const hasValue = Object.prototype.hasOwnProperty.call( mockQueryStore, diff --git a/packages/app/src/components/__tests__/DBRowSidePanel.spanLinks.test.tsx b/packages/app/src/components/__tests__/DBRowSidePanel.spanLinks.test.tsx index 28e8031937..346e29b1e9 100644 --- a/packages/app/src/components/__tests__/DBRowSidePanel.spanLinks.test.tsx +++ b/packages/app/src/components/__tests__/DBRowSidePanel.spanLinks.test.tsx @@ -24,7 +24,7 @@ jest.mock('nuqs', () => { const actual = jest.requireActual('nuqs'); return { ...actual, - // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useQueryState: (key: string, parser?: { defaultValue?: unknown }) => { const hasValue = Object.prototype.hasOwnProperty.call( mockQueryStore, @@ -56,7 +56,7 @@ const LINK = { const mockUseRowData = jest.fn(); jest.mock('../DBRowDataPanel', () => ({ __esModule: true, - // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useRowData: (args: unknown) => mockUseRowData(args), ROW_DATA_ALIASES: { DURATION_MS: '__hdx_duration', @@ -82,14 +82,14 @@ const TRACE_SOURCE = { jest.mock('@/source', () => ({ __esModule: true, getEventBody: () => undefined, - // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useSource: ({ id }: { id: string | null }) => id === 'trace-src' ? { data: TRACE_SOURCE } : { data: undefined }, })); jest.mock('../DBSessionPanel', () => ({ __esModule: true, - // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useSessionId: () => ({ rumSessionId: undefined, rumServiceName: undefined }), DBSessionPanel: () => null, })); diff --git a/packages/app/src/components/__tests__/DBRowSidePanel.spanLinksBreadcrumb.test.tsx b/packages/app/src/components/__tests__/DBRowSidePanel.spanLinksBreadcrumb.test.tsx index 2b7a988e3c..c6f2a8b421 100644 --- a/packages/app/src/components/__tests__/DBRowSidePanel.spanLinksBreadcrumb.test.tsx +++ b/packages/app/src/components/__tests__/DBRowSidePanel.spanLinksBreadcrumb.test.tsx @@ -38,7 +38,7 @@ const LINKED_SPAN_NAME = 'consume order.created'; const mockUseRowData = jest.fn(); jest.mock('../DBRowDataPanel', () => ({ __esModule: true, - // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useRowData: (args: unknown) => mockUseRowData(args), ROW_DATA_ALIASES: { DURATION_MS: '__hdx_duration', @@ -62,14 +62,14 @@ const TRACE_SOURCE = { jest.mock('@/source', () => ({ __esModule: true, getEventBody: () => undefined, - // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useSource: ({ id }: { id: string | null }) => id === 'trace-src' ? { data: TRACE_SOURCE } : { data: undefined }, })); jest.mock('../DBSessionPanel', () => ({ __esModule: true, - // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useSessionId: () => ({ rumSessionId: undefined, rumServiceName: undefined }), DBSessionPanel: () => null, })); diff --git a/packages/app/src/components/__tests__/DBRowSidePanel.staleStack.test.tsx b/packages/app/src/components/__tests__/DBRowSidePanel.staleStack.test.tsx index 471f270e14..e6f3314222 100644 --- a/packages/app/src/components/__tests__/DBRowSidePanel.staleStack.test.tsx +++ b/packages/app/src/components/__tests__/DBRowSidePanel.staleStack.test.tsx @@ -26,7 +26,7 @@ jest.mock('nuqs', () => { const actual = jest.requireActual('nuqs'); return { ...actual, - // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useQueryState: (key: string, parser?: { defaultValue?: unknown }) => { const hasValue = Object.prototype.hasOwnProperty.call( mockQueryStore, diff --git a/packages/app/src/hooks/__tests__/useDashboardKioskMode.test.tsx b/packages/app/src/hooks/__tests__/useDashboardKioskMode.test.tsx index 530c859712..69c7dbfbcf 100644 --- a/packages/app/src/hooks/__tests__/useDashboardKioskMode.test.tsx +++ b/packages/app/src/hooks/__tests__/useDashboardKioskMode.test.tsx @@ -6,7 +6,7 @@ jest.mock('nuqs', () => { const actual = jest.requireActual('nuqs'); return { ...actual, - // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useQueryState: () => [false, mockSetKioskMode], }; }); diff --git a/packages/app/src/hooks/__tests__/useSidePanelStack.test.tsx b/packages/app/src/hooks/__tests__/useSidePanelStack.test.tsx index 24fcd69cf1..8314a8a111 100644 --- a/packages/app/src/hooks/__tests__/useSidePanelStack.test.tsx +++ b/packages/app/src/hooks/__tests__/useSidePanelStack.test.tsx @@ -24,7 +24,7 @@ jest.mock('nuqs', () => { const actual = jest.requireActual('nuqs'); return { ...actual, - // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix + useQueryState: (key: string, parser?: { defaultValue?: unknown }) => { const hasValue = Object.prototype.hasOwnProperty.call( mockQueryStore, diff --git a/packages/app/src/searchFilters.test.ts b/packages/app/src/searchFilters.test.ts index 6478694347..ebff895a67 100644 --- a/packages/app/src/searchFilters.test.ts +++ b/packages/app/src/searchFilters.test.ts @@ -2,7 +2,14 @@ import { enableMapSet } from 'immer'; import { Filter } from '@hyperdx/common-utils/dist/types'; import { act, renderHook } from '@testing-library/react'; -import { parseQuery, useSearchPageFilterState } from '@/searchFilters'; +import { + mergeFiltersIntoWhereClause, + parseQuery, + replaceFiltersInWhereClause, + translateWhereClauseInQuery, + useSearchPageFilterState, + whereToFilters, +} from '@/searchFilters'; // Filter state stores values in Sets; the app enables immer's MapSet plugin at // startup, but this isolated hook test must enable it explicitly. @@ -192,3 +199,282 @@ describe('canonical key escaping at the persistence boundary', () => { }); }); }); + +describe('whereToFilters', () => { + it('derives SQL filters from a lucene where clause', () => { + expect(whereToFilters('host:"a"', 'lucene', new Set())).toEqual([ + { type: 'sql', condition: "host IN ('a')" }, + ]); + }); + + it('derives SQL filters from a sql where clause', () => { + expect(whereToFilters("host IN ('a', 'b')", 'sql', new Set())).toEqual([ + { type: 'sql', condition: "host IN ('a', 'b')" }, + ]); + }); + + it('projects only facet clauses, dropping free text', () => { + expect(whereToFilters('error 404 host:"a"', 'lucene', new Set())).toEqual([ + { type: 'sql', condition: "host IN ('a')" }, + ]); + }); + + it('quotes special-character columns via knownColumns', () => { + expect( + whereToFilters('service-name:"a"', 'lucene', new Set(['service-name'])), + ).toEqual([{ type: 'sql', condition: "`service-name` IN ('a')" }]); + }); + + it('emits map sub-keys in canonical bracket form', () => { + expect( + whereToFilters( + 'LogAttributes.host.name:"x"', + 'lucene', + new Set(['LogAttributes']), + ), + ).toEqual([ + { type: 'sql', condition: "LogAttributes['host.name'] IN ('x')" }, + ]); + }); +}); + +describe('replaceFiltersInWhereClause', () => { + it('rewrites a lucene facet clause while preserving free text', () => { + const where = 'host:"a" AND error'; + const filters: Filter[] = [{ type: 'sql', condition: "host IN ('b')" }]; + expect( + replaceFiltersInWhereClause(where, 'lucene', filters, new Set()), + ).toBe('error AND host:"b"'); + }); + + it('rewrites a sql facet conjunct while preserving other conjuncts', () => { + const where = "host IN ('a') AND foo = 1"; + const filters: Filter[] = [{ type: 'sql', condition: "host IN ('b')" }]; + expect(replaceFiltersInWhereClause(where, 'sql', filters, new Set())).toBe( + "foo = 1 AND host IN ('b')", + ); + }); + + it('emits a fresh clause when the where is empty', () => { + const filters: Filter[] = [{ type: 'sql', condition: "host IN ('b')" }]; + expect(replaceFiltersInWhereClause('', 'lucene', filters, new Set())).toBe( + 'host:"b"', + ); + }); + + it('wraps OR residual in parens before appending AND facet (Lucene OR semantics)', () => { + // When the residual (non-facet content) contains a top-level OR, appending + // new facet clauses with AND would change semantics without parenthesization. + // e.g. `error OR warn` residual + `level:"error"` → `(error OR warn) AND level:"error"` + // not `error OR warn AND level:"error"` (which parses as `error OR (warn AND level:"error")`) + const where = 'error OR warn'; + // No facet fields in this query, so replace with a fresh level filter. + // The residual `error OR warn` must be wrapped in parens. + const filters: Filter[] = [ + { type: 'sql', condition: "level IN ('error')" }, + ]; + const result = replaceFiltersInWhereClause( + where, + 'lucene', + filters, + new Set(), + ); + expect(result).toBe('(error OR warn) AND level:"error"'); + }); + + it('replaces lowercase sql facet (case-insensitive detection)', () => { + // lowercase `in` should still be recognised as a facet and replaced + const where = "host in ('a') AND foo = 1"; + const filters: Filter[] = [{ type: 'sql', condition: "host IN ('b')" }]; + expect(replaceFiltersInWhereClause(where, 'sql', filters, new Set())).toBe( + "foo = 1 AND host IN ('b')", + ); + }); +}); + +describe('mergeFiltersIntoWhereClause (legacy migration)', () => { + it('preserves the where text and appends a lucene filter', () => { + const filters: Filter[] = [ + { type: 'sql', condition: "SeverityText IN ('error')" }, + ]; + expect( + mergeFiltersIntoWhereClause( + 'ServiceName:"api"', + 'lucene', + filters, + new Set(), + ), + ).toBe('ServiceName:"api" AND SeverityText:"error"'); + }); + + it('preserves the where text and appends a sql filter', () => { + const filters: Filter[] = [ + { type: 'sql', condition: "SeverityText IN ('error')" }, + ]; + expect( + mergeFiltersIntoWhereClause( + "ServiceName = 'api'", + 'sql', + filters, + new Set(), + ), + ).toBe("ServiceName = 'api' AND SeverityText IN ('error')"); + }); + + it('parenthesizes a top-level OR in the where text before appending', () => { + const filters: Filter[] = [ + { type: 'sql', condition: "SeverityText IN ('error')" }, + ]; + expect( + mergeFiltersIntoWhereClause( + 'ServiceName:"api" OR ServiceName:"web"', + 'lucene', + filters, + new Set(), + ), + ).toBe('(ServiceName:"api" OR ServiceName:"web") AND SeverityText:"error"'); + }); + + it('keeps a filter already present in the where text (legacy AND semantics)', () => { + // Legacy `where` + `filters` were independent predicates ANDed at query + // time, so a filter referencing a column already in the where is not + // deduped — both clauses are preserved. + const filters: Filter[] = [ + { type: 'sql', condition: "ServiceName IN ('api')" }, + ]; + expect( + mergeFiltersIntoWhereClause( + 'ServiceName:"api"', + 'lucene', + filters, + new Set(), + ), + ).toBe('ServiceName:"api" AND ServiceName:"api"'); + }); +}); + +describe('translateWhereClauseInQuery (language switch)', () => { + it('returns the text unchanged when switching to the same language', () => { + expect( + translateWhereClauseInQuery( + 'ServiceName:"api"', + 'lucene', + 'lucene', + new Set(), + ), + ).toBe('ServiceName:"api"'); + }); + + it('translates lucene facets to SQL, preserving free text', () => { + expect( + translateWhereClauseInQuery( + 'error AND ServiceName:"api"', + 'lucene', + 'sql', + new Set(), + ), + ).toBe("error AND ServiceName IN ('api')"); + }); + + it('translates SQL facets to lucene', () => { + expect( + translateWhereClauseInQuery( + "ServiceName IN ('api')", + 'sql', + 'lucene', + new Set(), + ), + ).toBe('ServiceName:"api"'); + }); + + it('quotes SQL keys that need escaping in the target language', () => { + expect( + translateWhereClauseInQuery( + 'service-name:"a"', + 'lucene', + 'sql', + new Set(['service-name']), + ), + ).toBe("`service-name` IN ('a')"); + }); + + it('returns the text unchanged when there are no facets to translate', () => { + expect( + translateWhereClauseInQuery('error 404', 'lucene', 'sql', new Set()), + ).toBe('error 404'); + }); + + it('returns the text unchanged when the source does not parse', () => { + expect( + translateWhereClauseInQuery('service:', 'lucene', 'sql', new Set()), + ).toBe('service:'); + }); +}); + +describe('useSearchPageFilterState mutator identity stability', () => { + it('keeps every mutator reference stable across searchQuery changes when onFilterChange is stable', () => { + // Regression guard: the sidebar is memoized, so its props (the mutators) + // must keep stable identities while the user types. `searchQuery` changes + // per keystroke, but that alone must not re-create the mutators. + const onFilterChange = jest.fn(); + let props = { + searchQuery: EMPTY_SEARCH_QUERY, + onFilterChange, + knownColumns: new Set(), + }; + const { result, rerender } = renderHook(() => + useSearchPageFilterState(props), + ); + + const first = { + setFilterValue: result.current.setFilterValue, + setOnlyFilters: result.current.setOnlyFilters, + replaceFilterValue: result.current.replaceFilterValue, + setFilterRange: result.current.setFilterRange, + clearFilter: result.current.clearFilter, + clearAllFilters: result.current.clearAllFilters, + retainFiltersByColumns: result.current.retainFiltersByColumns, + }; + + props = { + ...props, + // A different searchQuery that still resolves to no facets, so the + // internal `filters` state is untouched and every mutator can be + // asserted stable. + searchQuery: [{ type: 'sql', condition: "Body LIKE '%error%'" }], + }; + rerender(); + + expect(result.current.setFilterValue).toBe(first.setFilterValue); + expect(result.current.setOnlyFilters).toBe(first.setOnlyFilters); + expect(result.current.replaceFilterValue).toBe(first.replaceFilterValue); + expect(result.current.setFilterRange).toBe(first.setFilterRange); + expect(result.current.clearFilter).toBe(first.clearFilter); + expect(result.current.clearAllFilters).toBe(first.clearAllFilters); + expect(result.current.retainFiltersByColumns).toBe( + first.retainFiltersByColumns, + ); + }); + + it('re-creates the mutators when onFilterChange changes identity', () => { + let onFilterChange = jest.fn(); + let props = { + searchQuery: EMPTY_SEARCH_QUERY, + onFilterChange, + knownColumns: new Set(), + }; + const { result, rerender } = renderHook(() => + useSearchPageFilterState(props), + ); + const firstSetFilterValue = result.current.setFilterValue; + + // A new onFilterChange identity (what happened per keystroke when + // handleSetFilters closed over the watched `where`) must invalidate the + // mutators. + onFilterChange = jest.fn(); + props = { ...props, onFilterChange }; + rerender(); + + expect(result.current.setFilterValue).not.toBe(firstSetFilterValue); + }); +}); diff --git a/packages/app/src/searchFilters.tsx b/packages/app/src/searchFilters.tsx index ac193b7f26..62b27dc1af 100644 --- a/packages/app/src/searchFilters.tsx +++ b/packages/app/src/searchFilters.tsx @@ -3,7 +3,10 @@ import produce from 'immer'; import { type FilterState, filtersToQuery, + mergeFilterStateIntoWhereClause, parseQuery, + parseWhereClauseToFilterState, + replaceFilterClauses, } from '@hyperdx/common-utils/dist/filters'; import type { Filter } from '@hyperdx/common-utils/dist/types'; @@ -35,7 +38,7 @@ export const escapeFilterStateKeys = ( }; // Convert valid SQL/persisted keys to clean FilterState keys. -const unescapeFilterStateKeys = (filters: FilterState): FilterState => { +export const unescapeFilterStateKeys = (filters: FilterState): FilterState => { const cleaned: FilterState = {}; for (const [key, value] of Object.entries(filters)) { cleaned[cleanClickHouseExpression(key)] = value; @@ -43,6 +46,112 @@ const unescapeFilterStateKeys = (filters: FilterState): FilterState => { return cleaned; }; +/** + * Derive the SQL `Filter[]` the search page's filter hook expects from a + * `where` clause string in either query language. The `where` text is the + * canonical form on the search page, so this adapter is what feeds the + * sidebar's FilterState: facet clauses are parsed out (`whereToFilters` is a + * lossy projection for facets only — free-text and complex content doesn't map + * to facets and is dropped here), escaped back to canonical SQL keys, and + * emitted as `filtersToQuery` filters so the hook's existing parse/unescape + * cycle restores clean keys for the sidebar. + */ +export const whereToFilters = ( + whereText: string, + whereLanguage: 'lucene' | 'sql', + knownColumns: Set, + dateTimeColumns?: ReadonlyMap, +): Filter[] => { + const state = parseWhereClauseToFilterState(whereText, whereLanguage); + const clean = + whereLanguage === 'sql' ? unescapeFilterStateKeys(state) : state; + return filtersToQuery(escapeFilterStateKeys(clean, knownColumns), { + dateTimeColumns, + }); +}; + +/** + * Rewrite a `where` clause string so its facet clauses reflect `filters` (the + * SQL `Filter[]` emitted by the filter hook's mutators), preserving unrelated + * free-text/complex content. Keys in `filters` are the canonical + * quoted/bracket ClickHouse form; they are unescaped to clean keys before the + * round-trip so `replaceFilterClauses` can match existing clauses. + */ +export const replaceFiltersInWhereClause = ( + whereText: string, + whereLanguage: 'lucene' | 'sql', + filters: Filter[], + knownColumns: Set, + dateTimeColumns?: ReadonlyMap, +): string => { + const cleanState = unescapeFilterStateKeys(parseQuery(filters).filters); + return replaceFilterClauses(whereText, whereLanguage, cleanState, { + escapeKey: + whereLanguage === 'sql' + ? key => toQuotedClickHouseKeyExpression(key, knownColumns) + : undefined, + dateTimeColumns, + }); +}; + +/** + * Append `filters` to a `where` clause while preserving the existing text + * verbatim. Unlike `replaceFiltersInWhereClause` (used for sidebar toggles, + * where the existing text's facet fields are owned by the FilterState and get + * replaced), this *merges*: the original clause and the new filters are both + * kept, ANDed together. + * + * This is the semantics required when migrating a legacy `where` + separate + * `filters` representation (independent predicates ANDed at query time) into + * the unified `where`, so neither side may be dropped. + */ +export const mergeFiltersIntoWhereClause = ( + whereText: string, + whereLanguage: 'lucene' | 'sql', + filters: Filter[], + knownColumns: Set, + dateTimeColumns?: ReadonlyMap, +): string => { + const cleanState = unescapeFilterStateKeys(parseQuery(filters).filters); + return mergeFilterStateIntoWhereClause(whereText, whereLanguage, cleanState, { + escapeKey: + whereLanguage === 'sql' + ? key => toQuotedClickHouseKeyExpression(key, knownColumns) + : undefined, + dateTimeColumns, + }); +}; + +/** + * Convert a `where` clause from one query language to another so the facet + * clauses transfer across a language switch instead of being dropped when the + * text is re-parsed in the new language. Facet clauses are re-emitted in the + * target language; non-facet content (free text, complex conditions) is + * preserved verbatim. If the text parses to no facets (or is unparseable) it is + * returned unchanged so we never clobber in-progress or facet-free input. + */ +export const translateWhereClauseInQuery = ( + whereText: string, + fromLanguage: 'lucene' | 'sql', + toLanguage: 'lucene' | 'sql', + knownColumns: Set, + dateTimeColumns?: ReadonlyMap, +): string => { + if (fromLanguage === toLanguage) return whereText; + const state = parseWhereClauseToFilterState(whereText, fromLanguage); + const cleanState = + fromLanguage === 'sql' ? unescapeFilterStateKeys(state) : state; + if (Object.keys(cleanState).length === 0) return whereText; + return replaceFilterClauses(whereText, fromLanguage, cleanState, { + emitLanguage: toLanguage, + escapeKey: + toLanguage === 'sql' + ? key => toQuotedClickHouseKeyExpression(key, knownColumns) + : undefined, + dateTimeColumns, + }); +}; + export const areFiltersEqual = (a: FilterState, b: FilterState) => { const aKeys = Object.keys(a); const bKeys = Object.keys(b); @@ -78,6 +187,10 @@ export const areFiltersEqual = (a: FilterState, b: FilterState) => { // filtersToQuery; re-export so existing `@/searchFilters` importers keep working. export { parseQuery }; +export { + getUnrepresentableWhereReason, + getWhereParseError, +} from '@hyperdx/common-utils/dist/filters'; export const useSearchPageFilterState = ({ searchQuery = [], onFilterChange, diff --git a/packages/common-utils/src/__tests__/filterRoundTrip.test.ts b/packages/common-utils/src/__tests__/filterRoundTrip.test.ts new file mode 100644 index 0000000000..b77366a71d --- /dev/null +++ b/packages/common-utils/src/__tests__/filterRoundTrip.test.ts @@ -0,0 +1,1258 @@ +import { + type FilterState, + filterStateToWhereClause, + getUnrepresentableWhereReason, + getWhereParseError, + mergeFilterStateIntoWhereClause, + parseWhereClauseToFilterState, + replaceFilterClauses, +} from '@/filters'; +import { parse } from '@/queryParser'; + +describe('filterStateToWhereClause (lucene)', () => { + it('emits a single included value', () => { + const state: FilterState = { + a: { included: new Set(['b']), excluded: new Set() }, + }; + expect(filterStateToWhereClause(state, { language: 'lucene' })).toBe( + 'a:"b"', + ); + }); + + it('emits parenthesized OR group for multiple included values', () => { + const state: FilterState = { + c: { included: new Set(['d', 'x']), excluded: new Set() }, + }; + expect(filterStateToWhereClause(state, { language: 'lucene' })).toBe( + '(c:"d" OR c:"x")', + ); + }); + + it('emits negated terms for excluded values', () => { + const state: FilterState = { + a: { + included: new Set(['b']), + excluded: new Set(['c']), + }, + }; + expect(filterStateToWhereClause(state, { language: 'lucene' })).toBe( + 'a:"b" AND -a:"c"', + ); + }); + + it('emits multiple excluded values joined with AND', () => { + const state: FilterState = { + a: { + included: new Set(), + excluded: new Set([true, false]), + }, + }; + expect(filterStateToWhereClause(state, { language: 'lucene' })).toBe( + '-a:"true" AND -a:"false"', + ); + }); + + it('emits boolean values as strings', () => { + const state: FilterState = { + isRootSpan: { + included: new Set([true]), + excluded: new Set(), + }, + }; + expect(filterStateToWhereClause(state, { language: 'lucene' })).toBe( + 'isRootSpan:"true"', + ); + }); + + it('escapes double quotes in values', () => { + const state: FilterState = { + message: { + included: new Set(['say "hello"']), + excluded: new Set(), + }, + }; + expect(filterStateToWhereClause(state, { language: 'lucene' })).toBe( + 'message:"say \\"hello\\""', + ); + }); + + it('escapes backslashes in values', () => { + const state: FilterState = { + FilePath: { + included: new Set(['C:\\path\\to\\file']), + excluded: new Set(), + }, + }; + expect(filterStateToWhereClause(state, { language: 'lucene' })).toBe( + 'FilePath:"C:\\\\path\\\\to\\\\file"', + ); + }); + + it('normalizes bracket-notation map keys to dot form', () => { + const state: FilterState = { + "LogAttributes['service.name']": { + included: new Set(['my-app']), + excluded: new Set(), + }, + }; + expect(filterStateToWhereClause(state, { language: 'lucene' })).toBe( + 'LogAttributes.service.name:"my-app"', + ); + }); + + it('escapes colons in map keys', () => { + const state: FilterState = { + "LogAttributes['foo:bar']": { + included: new Set(['value1']), + excluded: new Set(), + }, + }; + const emitted = filterStateToWhereClause(state, { + language: 'lucene', + }); + expect(emitted).toBe(String.raw`LogAttributes.foo\:bar:"value1"`); + expect(() => parse(emitted)).not.toThrow(); + }); + + it('emits range filters', () => { + const state: FilterState = { + duration: { + included: new Set(), + excluded: new Set(), + range: { min: 10, max: 500 }, + }, + }; + expect(filterStateToWhereClause(state, { language: 'lucene' })).toBe( + 'duration:[10 TO 500]', + ); + }); +}); + +describe('parseWhereClauseToFilterState (lucene)', () => { + it('parses a single included value', () => { + expect(parseWhereClauseToFilterState('a:"b"', 'lucene')).toEqual({ + a: { included: new Set(['b']), excluded: new Set() }, + }); + }); + + it('parses a negated term as excluded', () => { + expect(parseWhereClauseToFilterState('-a:"c"', 'lucene')).toEqual({ + a: { included: new Set(), excluded: new Set(['c']) }, + }); + }); + + it('parses an OR group into multiple included values', () => { + expect(parseWhereClauseToFilterState('(c:"d" OR c:"x")', 'lucene')).toEqual( + { + c: { included: new Set(['d', 'x']), excluded: new Set() }, + }, + ); + }); + + it('parses a range term', () => { + expect( + parseWhereClauseToFilterState('duration:[10 TO 500]', 'lucene'), + ).toEqual({ + duration: { + included: new Set(), + excluded: new Set(), + range: { min: 10, max: 500 }, + }, + }); + }); + + it('coerces boolean values', () => { + expect( + parseWhereClauseToFilterState('isRootSpan:"true"', 'lucene'), + ).toEqual({ + isRootSpan: { included: new Set([true]), excluded: new Set() }, + }); + }); + + it('ignores free-text phrases (implicit field)', () => { + expect(parseWhereClauseToFilterState('"error 404"', 'lucene')).toEqual({}); + }); + + it('ignores unquoted terms', () => { + expect(parseWhereClauseToFilterState('error 404', 'lucene')).toEqual({}); + }); + + it('decodes escaped colons in field names', () => { + expect( + parseWhereClauseToFilterState( + String.raw`LogAttributes.foo\:bar:"value1"`, + 'lucene', + ), + ).toEqual({ + 'LogAttributes.foo:bar': { + included: new Set(['value1']), + excluded: new Set(), + }, + }); + }); + + it('returns empty state for invalid lucene', () => { + expect(parseWhereClauseToFilterState('(((', 'lucene')).toEqual({}); + }); + + it('returns empty state for empty text', () => { + expect(parseWhereClauseToFilterState('', 'lucene')).toEqual({}); + }); + + it('round-trips emitted state', () => { + const state: FilterState = { + service: { included: new Set(['app', 'api']), excluded: new Set() }, + level: { included: new Set(), excluded: new Set(['debug']) }, + duration: { + included: new Set(), + excluded: new Set(), + range: { min: 1, max: 999 }, + }, + }; + const where = filterStateToWhereClause(state, { language: 'lucene' }); + expect(parseWhereClauseToFilterState(where, 'lucene')).toEqual(state); + }); +}); + +describe('replaceFilterClauses (lucene)', () => { + it('replaces one managed clause and re-emits the rest from the full state', () => { + const result = replaceFilterClauses( + 'foo:"x" AND host:"a" AND bar:"y"', + 'lucene', + { + host: { included: new Set(['b']), excluded: new Set() }, + foo: { included: new Set(['x']), excluded: new Set() }, + bar: { included: new Set(['y']), excluded: new Set() }, + }, + ); + expect(result).toBe('host:"b" AND foo:"x" AND bar:"y"'); + }); + + it('preserves free-text terms and re-emits facets', () => { + const result = replaceFilterClauses('error 404 AND host:"a"', 'lucene', { + host: { included: new Set(['b']), excluded: new Set() }, + }); + expect(result).toBe('error 404 AND host:"b"'); + }); + + it('preserves quoted free-text phrases', () => { + const result = replaceFilterClauses( + '"out of memory" AND host:"a"', + 'lucene', + { + host: { included: new Set(['b']), excluded: new Set() }, + }, + ); + expect(result).toBe('"out of memory" AND host:"b"'); + }); + + it('removes a clause when the new state drops the field', () => { + const result = replaceFilterClauses('host:"a" AND level:"info"', 'lucene', { + level: { included: new Set(['warn']), excluded: new Set() }, + }); + expect(result).toBe('level:"warn"'); + }); + + it('replaces all facets when given a full state', () => { + const result = replaceFilterClauses( + 'host:"a" AND level:"info" AND foo:"x"', + 'lucene', + { + host: { included: new Set(['b']), excluded: new Set() }, + level: { included: new Set(['warn']), excluded: new Set() }, + foo: { included: new Set(['x']), excluded: new Set() }, + }, + ); + expect(result).toBe('host:"b" AND level:"warn" AND foo:"x"'); + }); + + it('returns just the new clause for empty input', () => { + const result = replaceFilterClauses('', 'lucene', { + host: { included: new Set(['b']), excluded: new Set() }, + }); + expect(result).toBe('host:"b"'); + }); + + it('returns empty for a fully-managed input with empty state', () => { + const result = replaceFilterClauses('host:"a"', 'lucene', {}); + expect(result).toBe(''); + }); + + it('replaces invalid lucene with the new clause (sidebar click applies)', () => { + // Unparseable text (e.g. an incomplete query like `service:`) can't be + // rewritten in place; a sidebar click should still apply by replacing the + // broken text with the new state rather than silently no-oping. + const result = replaceFilterClauses('(((', 'lucene', { + host: { included: new Set(['b']), excluded: new Set() }, + }); + expect(result).toBe('host:"b"'); + }); + + it('preserves an OR group without dangling connectors', () => { + const result = replaceFilterClauses( + '(env:"prod" OR env:"staging") AND host:"a"', + 'lucene', + { + env: { included: new Set(['qa']), excluded: new Set() }, + host: { included: new Set(['b']), excluded: new Set() }, + }, + ); + expect(result).toBe('env:"qa" AND host:"b"'); + }); +}); + +describe('filterStateToWhereClause (sql)', () => { + it('emits IN clauses', () => { + const state: FilterState = { + host: { included: new Set(['a', 'b']), excluded: new Set() }, + }; + expect(filterStateToWhereClause(state, { language: 'sql' })).toBe( + "host IN ('a', 'b')", + ); + }); + + it('emits NOT IN clauses', () => { + const state: FilterState = { + host: { included: new Set(), excluded: new Set(['a']) }, + }; + expect(filterStateToWhereClause(state, { language: 'sql' })).toBe( + "host NOT IN ('a')", + ); + }); +}); + +describe('parseWhereClauseToFilterState (sql)', () => { + it('parses an IN clause', () => { + const state = parseWhereClauseToFilterState("host IN ('a', 'b')", 'sql'); + expect(Array.from(state.host?.included ?? [])).toEqual( + expect.arrayContaining(['a', 'b']), + ); + }); + + it('parses a NOT IN clause', () => { + const state = parseWhereClauseToFilterState("host NOT IN ('a')", 'sql'); + expect(Array.from(state.host?.excluded ?? [])).toEqual(['a']); + }); + + it('parses a BETWEEN clause into a range', () => { + const state = parseWhereClauseToFilterState( + 'duration BETWEEN 10 AND 500', + 'sql', + ); + expect(state.duration?.range).toEqual({ min: 10, max: 500 }); + }); +}); + +describe('replaceFilterClauses (sql)', () => { + it('replaces a facet conjunct and preserves other conjuncts', () => { + const result = replaceFilterClauses( + "host IN ('a') AND foo = 'bar'", + 'sql', + { + host: { included: new Set(['b']), excluded: new Set() }, + }, + ); + expect(result).toBe("foo = 'bar' AND host IN ('b')"); + }); + + it('handles BETWEEN conjuncts', () => { + const result = replaceFilterClauses( + 'duration BETWEEN 1 AND 100 AND foo = 1', + 'sql', + { + duration: { + included: new Set(), + excluded: new Set(), + range: { min: 10, max: 50 }, + }, + }, + ); + expect(result).toBe('foo = 1 AND duration BETWEEN 10 AND 50'); + }); + + it('preserves an IN value containing the separator', () => { + const result = replaceFilterClauses( + "host IN ('a AND b') AND foo = 'x'", + 'sql', + { + host: { included: new Set(['c']), excluded: new Set() }, + }, + ); + expect(result).toBe("foo = 'x' AND host IN ('c')"); + }); + + it('removes a facet conjunct when the new state drops the field', () => { + const result = replaceFilterClauses( + "host IN ('a') AND level = 'info'", + 'sql', + {}, + ); + expect(result).toBe("level = 'info'"); + }); + + it('returns the new clause for empty input', () => { + const result = replaceFilterClauses('', 'sql', { + host: { included: new Set(['b']), excluded: new Set() }, + }); + expect(result).toBe("host IN ('b')"); + }); +}); + +// Regression tests for the 10 WHERE-string generation bugs + +describe('NOT prefix is preserved through replaceFilterClauses', () => { + it('keeps the NOT prefix when a sibling facet clause is replaced', () => { + // Original: NOT term AND ServiceName:"api" + // Click sidebar to change ServiceName → "accounting" + // Must NOT silently strip the `NOT` operator. + const result = replaceFilterClauses( + 'NOT term AND ServiceName:"api"', + 'lucene', + { + ServiceName: { included: new Set(['accounting']), excluded: new Set() }, + }, + ); + expect(result).toBe('NOT term AND ServiceName:"accounting"'); + }); +}); + +describe('cross-field OR is not silently converted to AND', () => { + it('preserves a cross-field OR query as-is when a different facet is added', () => { + // ServiceName:"api" OR SeverityText:"error" cannot be represented as a + // FilterState (AND semantics), so the whole OR must be left untouched. + const result = replaceFilterClauses( + 'ServiceName:"api" OR SeverityText:"error"', + 'lucene', + { + // We are NOT replacing ServiceName or SeverityText — we add a third field. + level: { included: new Set(['warn']), excluded: new Set() }, + }, + ); + // The cross-field OR is preserved; the new clause is ANDed after. + expect(result).toBe( + '(ServiceName:"api" OR SeverityText:"error") AND level:"warn"', + ); + }); + + it('treats each side of a cross-field OR as unmanaged (no field collected)', () => { + // parseWhereClauseToFilterState should return empty for cross-field OR + // because neither side alone represents a reproducible facet state. + const state = parseWhereClauseToFilterState( + 'ServiceName:"api" OR SeverityText:"error"', + 'lucene', + ); + expect(state).toEqual({}); + }); +}); + +describe('field-group syntax ServiceName:("api" OR "web") round-trips correctly', () => { + it('parses a field group into the correct FilterState', () => { + const state = parseWhereClauseToFilterState( + 'ServiceName:("api" OR "web")', + 'lucene', + ); + expect(state).toEqual({ + ServiceName: { included: new Set(['api', 'web']), excluded: new Set() }, + }); + }); + + it('replaces a field-group clause without duplicating or dropping the field name', () => { + // Bug: was dropping ServiceName: and emitting ("api" OR "web") as free-text, + // then also appending the new clause → duplication + full-text search. + const result = replaceFilterClauses( + 'ServiceName:("api" OR "web") AND term', + 'lucene', + { + ServiceName: { + included: new Set(['api', 'web', 'admin']), + excluded: new Set(), + }, + }, + ); + // `term` (free-text) preserved; ServiceName clause replaced (no duplication). + expect(result).toBe( + 'term AND (ServiceName:"api" OR ServiceName:"web" OR ServiceName:"admin")', + ); + }); +}); + +describe('range clause at source offset 0 is not dropped', () => { + it('replaces a range that starts at the beginning of the query string', () => { + // Duration:[* TO 100] starts at offset 0 → fieldLocation.start.offset === 0, + // which is falsy and was causing the span to be null (clause silently dropped). + const result = replaceFilterClauses( + 'Duration:[0 TO 100] AND ServiceName:"api"', + 'lucene', + { + Duration: { + included: new Set(), + excluded: new Set(), + range: { min: 0, max: 50 }, + }, + ServiceName: { included: new Set(['web']), excluded: new Set() }, + }, + ); + expect(result).toBe('Duration:[0 TO 50] AND ServiceName:"web"'); + }); + + it('parses a range at offset 0 into FilterState correctly', () => { + const state = parseWhereClauseToFilterState( + 'Duration:[0 TO 100]', + 'lucene', + ); + expect(state).toEqual({ + Duration: { + included: new Set(), + excluded: new Set(), + range: { min: 0, max: 100 }, + }, + }); + }); +}); + +describe('exclusive range bounds are preserved', () => { + it('emits exclusive {…} brackets when the range has inclusive: none', () => { + const state: FilterState = { + Duration: { + included: new Set(), + excluded: new Set(), + range: { min: 10, max: 20, inclusive: 'none' }, + }, + }; + expect(filterStateToWhereClause(state, { language: 'lucene' })).toBe( + 'Duration:{10 TO 20}', + ); + }); + + it('round-trips an exclusive range without converting to inclusive', () => { + const result = replaceFilterClauses('Duration:{10 TO 20}', 'lucene', { + Duration: { + included: new Set(), + excluded: new Set(), + range: { min: 10, max: 20, inclusive: 'none' }, + }, + }); + expect(result).toBe('Duration:{10 TO 20}'); + }); + + it('parses an exclusive range and stores inclusive: none', () => { + const state = parseWhereClauseToFilterState( + 'Duration:{10 TO 20}', + 'lucene', + ); + expect(state.Duration?.range).toEqual({ + min: 10, + max: 20, + inclusive: 'none', + }); + }); + + it('emits left-exclusive {min TO max] for inclusive: right', () => { + const state: FilterState = { + score: { + included: new Set(), + excluded: new Set(), + range: { min: 0, max: 100, inclusive: 'right' }, + }, + }; + expect(filterStateToWhereClause(state, { language: 'lucene' })).toBe( + 'score:{0 TO 100]', + ); + }); +}); + +describe('proximity/boost modifiers are not stripped', () => { + it('preserves a proximity modifier when a sibling facet is replaced', () => { + // msg:"hello"~2 should remain untouched when ServiceName is changed. + const result = replaceFilterClauses( + 'msg:"hello"~2 AND ServiceName:"api"', + 'lucene', + { + ServiceName: { included: new Set(['web']), excluded: new Set() }, + }, + ); + expect(result).toBe('msg:"hello"~2 AND ServiceName:"web"'); + }); + + it('preserves a boost modifier when a sibling facet is replaced', () => { + const result = replaceFilterClauses( + 'title:"report"^3 AND level:"info"', + 'lucene', + { + level: { included: new Set(['warn']), excluded: new Set() }, + }, + ); + expect(result).toBe('title:"report"^3 AND level:"warn"'); + }); + + it('does not collect a proximity term into FilterState', () => { + // A proximity term cannot be faithfully represented; it should be skipped. + const state = parseWhereClauseToFilterState( + 'msg:"hello"~2 AND ServiceName:"api"', + 'lucene', + ); + expect(state).toEqual({ + ServiceName: { included: new Set(['api']), excluded: new Set() }, + }); + }); +}); + +describe('modifier terms and negated ranges do not survive alongside new clause', () => { + it('replaces a plain sibling clause when the field also has a modifier term', () => { + // msg:"hello"~2 cannot round-trip into FilterState, but msg:"world" can. + // When the sidebar sets msg:"goodbye", msg:"world" must be removed so it + // does not pile up alongside the new clause. + const result = replaceFilterClauses( + 'msg:"hello"~2 AND msg:"world" AND ServiceName:"api"', + 'lucene', + { + msg: { included: new Set(['goodbye']), excluded: new Set() }, + ServiceName: { included: new Set(['api']), excluded: new Set() }, + }, + ); + // msg:"world" is pruned; msg:"hello"~2 is preserved verbatim (unmanageable). + expect(result).toBe('msg:"hello"~2 AND msg:"goodbye" AND ServiceName:"api"'); + }); + + it('preserves a modifier-only field verbatim and appends the new clause', () => { + // If there is no plain sibling, the modifier term is kept and the new + // clause is appended — the sidebar click still applies. + const result = replaceFilterClauses( + 'msg:"hello"~2 AND ServiceName:"api"', + 'lucene', + { + msg: { included: new Set(['goodbye']), excluded: new Set() }, + ServiceName: { included: new Set(['api']), excluded: new Set() }, + }, + ); + expect(result).toBe('msg:"hello"~2 AND msg:"goodbye" AND ServiceName:"api"'); + }); + + it('replaces a plain range clause when the field also has a negated range', () => { + // NOT duration:[10 TO 20] cannot round-trip, but duration:[0 TO 5] can. + // When the sidebar sets a new range for duration, duration:[0 TO 5] must + // be removed so it does not duplicate alongside the new range. + const result = replaceFilterClauses( + 'NOT duration:[10 TO 20] AND duration:[0 TO 5]', + 'lucene', + { + duration: { + included: new Set(), + excluded: new Set(), + range: { min: 1, max: 3 }, + }, + }, + ); + // duration:[0 TO 5] is pruned; NOT duration:[10 TO 20] is preserved. + expect(result).toBe('NOT duration:[10 TO 20] AND duration:[1 TO 3]'); + }); +}); + +describe('unquoted field term is replaced (not duplicated) by sidebar click', () => { + it('replaces an unquoted field term when the sidebar emits a quoted one', () => { + // level:error (unquoted) → sidebar adds level:"warn" + // Was: level:error AND level:"warn" → zero rows (two conflicting conditions) + // Fixed: level:"warn" + const result = replaceFilterClauses('level:error', 'lucene', { + level: { included: new Set(['warn']), excluded: new Set() }, + }); + expect(result).toBe('level:"warn"'); + }); + + it('does not add a quoted duplicate alongside an unquoted term', () => { + const result = replaceFilterClauses( + 'level:error AND ServiceName:"api"', + 'lucene', + { + level: { included: new Set(['warn']), excluded: new Set() }, + ServiceName: { included: new Set(['api']), excluded: new Set() }, + }, + ); + expect(result).toBe('level:"warn" AND ServiceName:"api"'); + }); +}); + +describe('attribute keys with special chars are properly escaped', () => { + it('escapes ( and ) in attribute key names', () => { + const state: FilterState = { + "LogAttributes['a(b)']": { + included: new Set(['value']), + excluded: new Set(), + }, + }; + const emitted = filterStateToWhereClause(state, { language: 'lucene' }); + // Must be parseable (not throw) and contain the correct field. + expect(() => parse(emitted)).not.toThrow(); + expect(emitted).toContain('"value"'); + }); + + it('escapes [ and ] in attribute key names', () => { + const state: FilterState = { + "LogAttributes['arr[0]']": { + included: new Set(['x']), + excluded: new Set(), + }, + }; + const emitted = filterStateToWhereClause(state, { language: 'lucene' }); + expect(() => parse(emitted)).not.toThrow(); + }); + + it('escapes { in attribute key names', () => { + const state: FilterState = { + "LogAttributes['{key}']": { + included: new Set(['v']), + excluded: new Set(), + }, + }; + const emitted = filterStateToWhereClause(state, { language: 'lucene' }); + expect(() => parse(emitted)).not.toThrow(); + }); + + it('escapes spaces in attribute key names', () => { + const state: FilterState = { + "LogAttributes['my key']": { + included: new Set(['v']), + excluded: new Set(), + }, + }; + const emitted = filterStateToWhereClause(state, { language: 'lucene' }); + expect(() => parse(emitted)).not.toThrow(); + }); +}); + +describe('closing paren inside a quoted value does not break top-level OR detection', () => { + it('wraps the residual in parens when it has a top-level OR with a paren inside a quoted value', () => { + // "timeout)" OR "error" → the ) inside the quoted string must NOT be counted + // as a closing paren for depth tracking. + const result = replaceFilterClauses('"timeout)" OR "error"', 'lucene', { + level: { included: new Set(['x']), excluded: new Set() }, + }); + // The residual "timeout)" OR "error" has a top-level OR → must be wrapped. + expect(result).toBe('("timeout)" OR "error") AND level:"x"'); + }); + + it('does not double-wrap when the residual is already parenthesized', () => { + const result = replaceFilterClauses('("a)" OR "b")', 'lucene', { + level: { included: new Set(['x']), excluded: new Set() }, + }); + // Already parenthesized; top-level OR is inside parens → no extra wrap. + expect(result).toBe('("a)" OR "b") AND level:"x"'); + }); +}); + +describe('no extra spaces accumulate on repeated sidebar clicks', () => { + it('does not grow extra spaces inside the OR group after multiple interactions', () => { + // First click + const step1 = replaceFilterClauses('term1 OR term2', 'lucene', { + ServiceName: { included: new Set(['api']), excluded: new Set() }, + }); + // step1 should be "(term1 OR term2) AND ServiceName:\"api\"" + expect(step1).toBe('(term1 OR term2) AND ServiceName:"api"'); + + // Second click (simulates toggling another value) + const step2 = replaceFilterClauses(step1, 'lucene', { + ServiceName: { included: new Set(['web']), excluded: new Set() }, + }); + expect(step2).toBe('(term1 OR term2) AND ServiceName:"web"'); + + // Third click + const step3 = replaceFilterClauses(step2, 'lucene', { + ServiceName: { included: new Set(['admin']), excluded: new Set() }, + }); + expect(step3).toBe('(term1 OR term2) AND ServiceName:"admin"'); + + // Fourth click — must still be the same paren group with no extra spaces. + const step4 = replaceFilterClauses(step3, 'lucene', { + ServiceName: { + included: new Set(['api', 'admin']), + excluded: new Set(), + }, + }); + expect(step4).toBe( + '(term1 OR term2) AND (ServiceName:"api" OR ServiceName:"admin")', + ); + }); + + it('does not add trailing space to individual terms when re-joining', () => { + const step1 = replaceFilterClauses('term1 OR term2', 'lucene', { + level: { included: new Set(['warn']), excluded: new Set() }, + }); + // Should be exactly "(term1 OR term2) AND level:\"warn\"" — no extra spaces + expect(step1).not.toMatch(/term1 {2,}OR|OR {2,}term2/); + expect(step1).toBe('(term1 OR term2) AND level:"warn"'); + }); +}); + +describe('top-level OR residual is parenthesized before AND join', () => { + it('wraps the residual in parens so the facet only applies to the whole OR', () => { + const result = replaceFilterClauses( + "ServiceName = 'a' OR ServiceName = 'b'", + 'sql', + { + ServiceName: { included: new Set(['b']), excluded: new Set() }, + }, + ); + expect(result).toBe( + "(ServiceName = 'a' OR ServiceName = 'b') AND ServiceName IN ('b')", + ); + }); + + it('handles lowercase or operator', () => { + const result = replaceFilterClauses( + "ServiceName = 'a' or ServiceName = 'b'", + 'sql', + { + SeverityText: { included: new Set(['error']), excluded: new Set() }, + }, + ); + expect(result).toBe( + "(ServiceName = 'a' or ServiceName = 'b') AND SeverityText IN ('error')", + ); + }); + + it('does not double-wrap when the OR is already inside parens', () => { + const result = replaceFilterClauses( + "ServiceName = 'a' AND (level = 'info' OR level = 'warn')", + 'sql', + { + host: { included: new Set(['web']), excluded: new Set() }, + }, + ); + expect(result).toBe( + "ServiceName = 'a' AND (level = 'info' OR level = 'warn') AND host IN ('web')", + ); + }); + + it('does not treat an OR inside a quoted value as top-level', () => { + const result = replaceFilterClauses("msg = 'a OR b'", 'sql', { + host: { included: new Set(['web']), excluded: new Set() }, + }); + expect(result).toBe("msg = 'a OR b' AND host IN ('web')"); + }); +}); + +describe('line comments do not swallow appended facets', () => { + it('re-appends a trailing -- comment after the new predicate', () => { + const result = replaceFilterClauses( + "ServiceName = 'a' -- temp note", + 'sql', + { + ServiceName: { included: new Set(['b']), excluded: new Set() }, + }, + ); + expect(result).toBe( + "ServiceName = 'a' AND ServiceName IN ('b') -- temp note", + ); + }); + + it('does not treat a commented-out AND as a conjunct separator', () => { + const result = replaceFilterClauses( + "ServiceName = 'a' -- note AND should stay commented", + 'sql', + { + host: { included: new Set(['web']), excluded: new Set() }, + }, + ); + expect(result).toBe( + "ServiceName = 'a' AND host IN ('web') -- note AND should stay commented", + ); + }); + + it('leaves a -- inside a quoted value untouched', () => { + const result = replaceFilterClauses("msg = 'a -- b'", 'sql', { + host: { included: new Set(['web']), excluded: new Set() }, + }); + expect(result).toBe("msg = 'a -- b' AND host IN ('web')"); + }); + + it('re-appends a block comment after the new predicate', () => { + const result = replaceFilterClauses( + "ServiceName = 'a' /* temp note */", + 'sql', + { + ServiceName: { included: new Set(['b']), excluded: new Set() }, + }, + ); + expect(result).toBe( + "ServiceName = 'a' AND ServiceName IN ('b') /* temp note */", + ); + }); +}); + +describe('unbalanced paren inside a string and quotes in backtick keys', () => { + it('does not pile up conjuncts when a string contains an unmatched (', () => { + const step1 = replaceFilterClauses("msg = 'x AND y IN ('", 'sql', { + ServiceName: { included: new Set(['a']), excluded: new Set() }, + }); + expect(step1).toBe("msg = 'x AND y IN (' AND ServiceName IN ('a')"); + + // Second click must replace, not duplicate. + const step2 = replaceFilterClauses(step1, 'sql', { + ServiceName: { included: new Set(['a', 'b']), excluded: new Set() }, + }); + expect(step2).toBe("msg = 'x AND y IN (' AND ServiceName IN ('a', 'b')"); + }); + + it('treats a single quote inside a backtick identifier as literal', () => { + const step1 = replaceFilterClauses("`it's` = 1", 'sql', { + ServiceName: { included: new Set(['a']), excluded: new Set() }, + }); + expect(step1).toBe("`it's` = 1 AND ServiceName IN ('a')"); + + // Second click must replace, not duplicate. + const step2 = replaceFilterClauses(step1, 'sql', { + ServiceName: { included: new Set(['a', 'b']), excluded: new Set() }, + }); + expect(step2).toBe("`it's` = 1 AND ServiceName IN ('a', 'b')"); + }); +}); + +describe('backticked column facet replaces cleanly on repeat clicks', () => { + it('does not duplicate a backticked key with a hyphen', () => { + const escapeKey = (key: string) => `\`${key}\``; + const step1 = replaceFilterClauses( + '', + 'sql', + { + 'service-name': { included: new Set(['a']), excluded: new Set() }, + }, + { escapeKey }, + ); + expect(step1).toBe("`service-name` IN ('a')"); + + const step2 = replaceFilterClauses( + step1, + 'sql', + { + 'service-name': { included: new Set(['a', 'b']), excluded: new Set() }, + }, + { escapeKey }, + ); + expect(step2).toBe("`service-name` IN ('a', 'b')"); + }); +}); + +describe('IN (SELECT ...) subqueries are preserved, not treated as facets', () => { + it('does not destroy a subquery when a different facet is added', () => { + const result = replaceFilterClauses( + 'ServiceName IN (SELECT name FROM t) AND foo = 1', + 'sql', + { + ServiceName: { included: new Set(['b']), excluded: new Set() }, + }, + ); + expect(result).toBe( + "ServiceName IN (SELECT name FROM t) AND foo = 1 AND ServiceName IN ('b')", + ); + }); + + it('does not render a checkbox for a subquery value', () => { + const state = parseWhereClauseToFilterState( + 'ServiceName IN (SELECT name FROM t) AND foo = 1', + 'sql', + ); + expect(state).toEqual({}); + }); +}); + +describe('repeated SQL predicates for the same field are not merged into a union', () => { + it('leaves both conjuncts untouched when the same key appears twice', () => { + // host IN ('a') AND host IN ('b') is an intersection — the user wrote it + // intentionally. Merging into host IN ('a', 'b') would silently change the + // semantics to a union. The two conjuncts must be preserved verbatim. + const result = replaceFilterClauses( + "host IN ('a') AND host IN ('b')", + 'sql', + { + host: { included: new Set(['c']), excluded: new Set() }, + }, + ); + expect(result).toBe( + "host IN ('a') AND host IN ('b') AND host IN ('c')", + ); + }); + + it('still replaces a key that appears exactly once', () => { + const result = replaceFilterClauses( + "host IN ('a') AND level IN ('error')", + 'sql', + { + host: { included: new Set(['b']), excluded: new Set() }, + level: { included: new Set(['warn']), excluded: new Set() }, + }, + ); + expect(result).toBe("host IN ('b') AND level IN ('warn')"); + }); + + it('leaves a duplicate key untouched while still replacing a single-occurrence key', () => { + const result = replaceFilterClauses( + "host IN ('a') AND host IN ('b') AND level IN ('error')", + 'sql', + { + level: { included: new Set(['warn']), excluded: new Set() }, + }, + ); + expect(result).toBe( + "host IN ('a') AND host IN ('b') AND level IN ('warn')", + ); + }); +}); + +describe('mergeFilterStateIntoWhereClause preserves the existing where text', () => { + const errorSeverity: FilterState = { + SeverityText: { included: new Set(['error']), excluded: new Set() }, + }; + + it('appends lucene filters to a lucene where clause', () => { + expect( + mergeFilterStateIntoWhereClause( + 'ServiceName:"api"', + 'lucene', + errorSeverity, + ), + ).toBe('ServiceName:"api" AND SeverityText:"error"'); + }); + + it('appends SQL filters to a SQL where clause', () => { + expect( + mergeFilterStateIntoWhereClause("ServiceName = 'api'", 'sql', { + SeverityText: { included: new Set(['error']), excluded: new Set() }, + }), + ).toBe("ServiceName = 'api' AND SeverityText IN ('error')"); + }); + + it('parenthesizes a top-level OR in the existing lucene text before appending', () => { + expect( + mergeFilterStateIntoWhereClause( + 'a:"1" OR b:"2"', + 'lucene', + errorSeverity, + ), + ).toBe('(a:"1" OR b:"2") AND SeverityText:"error"'); + }); + + it('does not double-wrap an already-parenthesized lucene OR', () => { + expect( + mergeFilterStateIntoWhereClause('(a:"1" OR b:"2")', 'lucene', { + SeverityText: { included: new Set(['error']), excluded: new Set() }, + }), + ).toBe('(a:"1" OR b:"2") AND SeverityText:"error"'); + }); + + it('parenthesizes a top-level OR in the existing SQL text before appending', () => { + expect( + mergeFilterStateIntoWhereClause( + "ServiceName = 'a' OR ServiceName = 'b'", + 'sql', + errorSeverity, + ), + ).toBe( + "(ServiceName = 'a' OR ServiceName = 'b') AND SeverityText IN ('error')", + ); + }); + + it('re-appends SQL comments after the appended predicate', () => { + expect( + mergeFilterStateIntoWhereClause( + "ServiceName = 'a' -- temp note", + 'sql', + errorSeverity, + ), + ).toBe("ServiceName = 'a' AND SeverityText IN ('error') -- temp note"); + }); + + it('uses escapeKey for the emitted SQL clauses while preserving the original text', () => { + const escapeKey = (key: string) => `\`${key}\``; + expect( + mergeFilterStateIntoWhereClause( + "`service-name` = 'a'", + 'sql', + { 'service-name': { included: new Set(['b']), excluded: new Set() } }, + { escapeKey }, + ), + ).toBe("`service-name` = 'a' AND `service-name` IN ('b')"); + }); + + it('returns only the emitted state when the where text is empty', () => { + expect(mergeFilterStateIntoWhereClause('', 'lucene', errorSeverity)).toBe( + 'SeverityText:"error"', + ); + }); + + it('returns the trimmed where text when the state emits nothing', () => { + expect(mergeFilterStateIntoWhereClause(' a:"1" ', 'lucene', {})).toBe( + 'a:"1"', + ); + }); +}); + +describe('replaceFilterClauses emitLanguage (query-language translation)', () => { + const serviceApi: FilterState = { + ServiceName: { included: new Set(['api']), excluded: new Set() }, + }; + + it('translates lucene facets to SQL while preserving free text', () => { + expect( + replaceFilterClauses( + 'ServiceName:"api" AND error', + 'lucene', + serviceApi, + { emitLanguage: 'sql' }, + ), + ).toBe("error AND ServiceName IN ('api')"); + }); + + it('translates SQL facets to lucene', () => { + expect( + replaceFilterClauses("ServiceName IN ('api')", 'sql', serviceApi, { + emitLanguage: 'lucene', + }), + ).toBe('ServiceName:"api"'); + }); + + it('wraps a cross-field OR residual in parens before the SQL clause', () => { + expect( + replaceFilterClauses( + 'a:"1" OR b:"2"', + 'lucene', + { c: { included: new Set(['3']), excluded: new Set() } }, + { emitLanguage: 'sql' }, + ), + ).toBe('(a:"1" OR b:"2") AND c IN (\'3\')'); + }); + + it('emits SQL ranges via escapeKey when translating lucene to SQL', () => { + const escapeKey = (key: string) => `\`${key}\``; + expect( + replaceFilterClauses( + 'service-name:"a"', + 'lucene', + { 'service-name': { included: new Set(['b']), excluded: new Set() } }, + { emitLanguage: 'sql', escapeKey }, + ), + ).toBe("`service-name` IN ('b')"); + }); + + it('returns the where text unchanged when there is nothing to translate', () => { + expect( + replaceFilterClauses( + 'error 404', + 'lucene', + {}, + { + emitLanguage: 'sql', + }, + ), + ).toBe('error 404'); + }); +}); + +describe('getWhereParseError', () => { + it('returns null for empty, valid, and SQL input', () => { + expect(getWhereParseError('', 'lucene')).toBeNull(); + expect(getWhereParseError('ServiceName:"api"', 'lucene')).toBeNull(); + expect(getWhereParseError("ServiceName = 'api'", 'sql')).toBeNull(); + }); + + it('returns a message for unparseable lucene', () => { + expect(getWhereParseError('service:', 'lucene')).not.toBeNull(); + expect(getWhereParseError('(((', 'lucene')).not.toBeNull(); + }); +}); + +describe('getUnrepresentableWhereReason', () => { + it('flags a cross-field OR', () => { + expect( + getUnrepresentableWhereReason( + 'ServiceName:"api" OR SeverityText:"error"', + 'lucene', + ), + ).not.toBeNull(); + }); + + it('flags an OR NOT across fields', () => { + expect( + getUnrepresentableWhereReason('a:"1" OR NOT b:"2"', 'lucene'), + ).not.toBeNull(); + }); + + it('does not flag AND or same-field OR', () => { + expect( + getUnrepresentableWhereReason( + 'ServiceName:"api" AND SeverityText:"error"', + 'lucene', + ), + ).toBeNull(); + expect( + getUnrepresentableWhereReason('a:"1" OR a:"2"', 'lucene'), + ).toBeNull(); + }); + + it('does not flag SQL or unparseable input', () => { + expect( + getUnrepresentableWhereReason( + "ServiceName = 'api' OR SeverityText = 'error'", + 'sql', + ), + ).toBeNull(); + expect(getUnrepresentableWhereReason('service:', 'lucene')).toBeNull(); + }); +}); + +describe('lucene NOT keyword negation', () => { + it('parses NOT field:"value" as excluded', () => { + expect( + parseWhereClauseToFilterState('NOT ServiceName:"api"', 'lucene'), + ).toEqual({ + ServiceName: { included: new Set(), excluded: new Set(['api']) }, + }); + }); + + it('parses term AND NOT field:"value" as excluded', () => { + expect( + parseWhereClauseToFilterState('term AND NOT ServiceName:"api"', 'lucene'), + ).toEqual({ + ServiceName: { included: new Set(), excluded: new Set(['api']) }, + }); + }); + + it('parses AND NOT on a second field', () => { + expect( + parseWhereClauseToFilterState('a:"1" AND NOT b:"2"', 'lucene'), + ).toEqual({ + a: { included: new Set(['1']), excluded: new Set() }, + b: { included: new Set(), excluded: new Set(['2']) }, + }); + }); + + it('parses NOT prefix on a binary left subtree', () => { + expect( + parseWhereClauseToFilterState('NOT a:"1" AND b:"2"', 'lucene'), + ).toEqual({ + a: { included: new Set(), excluded: new Set(['1']) }, + b: { included: new Set(['2']), excluded: new Set() }, + }); + }); + + it('treats a double negation as included', () => { + expect(parseWhereClauseToFilterState('NOT -a:"1"', 'lucene')).toEqual({ + a: { included: new Set(['1']), excluded: new Set() }, + }); + }); + + it('does not collect a cross-field OR NOT as facets', () => { + expect( + parseWhereClauseToFilterState('a:"1" OR NOT b:"2"', 'lucene'), + ).toEqual({}); + }); + + it('round-trips NOT via excluded emission', () => { + const state = parseWhereClauseToFilterState( + 'term AND NOT ServiceName:"api"', + 'lucene', + ); + expect(filterStateToWhereClause(state, { language: 'lucene' })).toBe( + '-ServiceName:"api"', + ); + }); +}); diff --git a/packages/common-utils/src/__tests__/queryParser.test.ts b/packages/common-utils/src/__tests__/queryParser.test.ts index b6b0594914..bf42d6ff58 100644 --- a/packages/common-utils/src/__tests__/queryParser.test.ts +++ b/packages/common-utils/src/__tests__/queryParser.test.ts @@ -62,7 +62,7 @@ describe('CustomSchemaSQLSerializerV2 - json', () => { it('getColumnForField', async () => { const field1 = 'ResourceAttributesJSON.test'; const res1 = await serializer.getColumnForField(field1, {}); - expect(res1).toEqual({ + expect(res1).toMatchObject({ column: '', columnJSON: { number: @@ -75,7 +75,7 @@ describe('CustomSchemaSQLSerializerV2 - json', () => { }); const field2 = 'ResourceAttributesJSON.test.nest'; const res2 = await serializer.getColumnForField(field2, {}); - expect(res2).toEqual({ + expect(res2).toMatchObject({ column: '', columnJSON: { number: @@ -1776,7 +1776,7 @@ describe('CustomSchemaSQLSerializerV2 - Array and Nested Fields', () => { it('getColumnForField', async () => { const field1 = 'Events.Name'; const res1 = await serializer.getColumnForField(field1, {}); - expect(res1).toEqual({ + expect(res1).toMatchObject({ column: 'Events.Name', found: true, propertyType: JSDataType.String, @@ -1785,7 +1785,7 @@ describe('CustomSchemaSQLSerializerV2 - Array and Nested Fields', () => { const field2 = 'Events.Count'; const res2 = await serializer.getColumnForField(field2, {}); - expect(res2).toEqual({ + expect(res2).toMatchObject({ column: 'Events.Count', found: true, propertyType: JSDataType.Number, @@ -1794,7 +1794,7 @@ describe('CustomSchemaSQLSerializerV2 - Array and Nested Fields', () => { const field3 = 'Events.IsAvailable'; const res3 = await serializer.getColumnForField(field3, {}); - expect(res3).toEqual({ + expect(res3).toMatchObject({ column: 'Events.IsAvailable', found: true, propertyType: JSDataType.Bool, diff --git a/packages/common-utils/src/core/dateTimeValue.ts b/packages/common-utils/src/core/dateTimeValue.ts new file mode 100644 index 0000000000..f5a651326d --- /dev/null +++ b/packages/common-utils/src/core/dateTimeValue.ts @@ -0,0 +1,30 @@ +// Wrap a quoted string literal in a ClickHouse expression whose result type +// matches the date column's type. Shared by the SQL filter emitter +// (filters.ts) and the Lucene serializer's date-column equality/range +// rendering (queryParser.ts) so the two query paths produce byte-identical +// predicates for the same date value. +export const dateTimeValueExpr = ( + chType: string, + quotedValue: string, +): string => { + const dt64 = chType.match(/DateTime64\((\d+)/); + + if (dt64) { + return `parseDateTime64BestEffort(${quotedValue}, ${dt64[1]})`; + } + + if (/\bDateTime\b/.test(chType)) { + return `parseDateTimeBestEffort(${quotedValue})`; + } + + if (/\bDate32\b/.test(chType)) { + return `toDate32(${quotedValue})`; + } + + if (/\bDate\b/.test(chType)) { + return `toDate(${quotedValue})`; + } + + // Fallback for an unexpected type; DateTime64(9) covers the widest range. + return `parseDateTime64BestEffort(${quotedValue}, 9)`; +}; diff --git a/packages/common-utils/src/filters.ts b/packages/common-utils/src/filters.ts index 3bc425baea..54799b932e 100644 --- a/packages/common-utils/src/filters.ts +++ b/packages/common-utils/src/filters.ts @@ -1,14 +1,33 @@ +import type * as lucene from '@hyperdx/lucene'; import * as SQLParser from 'node-sql-parser'; +import { dateTimeValueExpr } from '@/core/dateTimeValue'; +import { parseKeyPath } from '@/core/metadata'; import { replaceJsonExpressions } from '@/core/utils'; -import { parse } from '@/queryParser'; +import { + decodeSpecialTokens, + isBinaryAST, + isLeftOnlyAST, + isNodeRangedTerm, + isNodeTerm, + parse, +} from '@/queryParser'; import { DashboardFilter, Filter } from '@/types'; export type FilterState = { [key: string]: { included: Set; excluded: Set; - range?: { min: number; max: number }; // For BETWEEN conditions + range?: { + min: number; + max: number; + /** + * Lucene range bracket style: 'both' ([min TO max]), 'none' ({min TO max}), + * 'left' ([min TO max}), 'right' ({min TO max]). + * Defaults to 'both' (fully inclusive) when not set. + */ + inclusive?: 'both' | 'none' | 'left' | 'right'; + }; }; }; @@ -16,31 +35,6 @@ const escapeString = (s: string) => { return s.replace(/\\/g, '\\\\').replace(/'/g, "''"); }; -// Wrap a quoted string literal in a ClickHouse expression whose result type -// matches the date column's type. -const dateTimeValueExpr = (chType: string, quotedValue: string): string => { - const dt64 = chType.match(/DateTime64\((\d+)/); - - if (dt64) { - return `parseDateTime64BestEffort(${quotedValue}, ${dt64[1]})`; - } - - if (/\bDateTime\b/.test(chType)) { - return `parseDateTimeBestEffort(${quotedValue})`; - } - - if (/\bDate32\b/.test(chType)) { - return `toDate32(${quotedValue})`; - } - - if (/\bDate\b/.test(chType)) { - return `toDate(${quotedValue})`; - } - - // Fallback for an unexpected type; DateTime64(9) covers the widest range. - return `parseDateTime64BestEffort(${quotedValue}, 9)`; -}; - export const filtersToQuery = ( filters: FilterState, { @@ -155,6 +149,1266 @@ export const serializeFilterState = (state: FilterState): string => { ); }; +/** The Lucene sentinel the parser uses for terms without an explicit field. */ +const IMPLICIT_FIELD = ''; + +/** Escape a value for use inside a Lucene quoted term ("...") */ +const escapeLuceneQuotedTerm = (s: string) => { + return s.replace(/\\/g, '\\\\').replace(/"/g, '\\"'); +}; + +/** + * Escape characters in a field name that would break Lucene syntax. + * + * Lucene treats several characters as syntax in field names / at the + * field:value boundary: + * - `\` – escape prefix (must come first so later replacements survive) + * - `:` – field/value separator + * - `(` `)` – grouping / field-group open/close + * - `"` – quoted-value delimiter + * - `{` `}` – exclusive range bracket + * - `[` `]` – inclusive range bracket + * - ` ` (space) – token separator + * + * Backslashes are escaped first so the `\X` sequences we insert later are + * not double-processed. The encoder's special-token rules (`encodeSpecialTokens`) + * handle `\:` → HDX_COLON so that colons survive the parser; the other + * characters are simply backslash-escaped in the raw Lucene text. + */ +const escapeLuceneFieldName = (key: string): string => + key + .replace(/\\/g, '\\\\') + .replace(/:/g, '\\:') + .replace(/\(/g, '\\(') + .replace(/\)/g, '\\)') + .replace(/"/g, '\\"') + .replace(/\{/g, '\\{') + .replace(/\}/g, '\\}') + .replace(/\[/g, '\\[') + .replace(/\]/g, '\\]') + .replace(/ /g, '\\ '); + +/** + * Render a FilterState as a single `where` clause string in the given query + * language. + * + * - `sql`: emits the same ` IN (...) / NOT IN (...) / BETWEEN ... AND ...` + * predicates `filtersToQuery` produces, remapping every clean key through + * `escapeKey` first (the app passes `toQuotedClickHouseKeyExpression`). + * - `lucene`: emits `field:"value"` terms, parenthesized `OR` groups for + * multi-value includes, negated `-field:"value"` terms for excludes, and + * `field:[min TO max]` ranges — the grammar that `parseWhereClauseToFilterState` + * reads back. + * + * This is the emission half of the search page's unified + * sidebar-⇄-query-input representation. Keys are the clean (dot-form) keys the + * sidebar uses. + */ +export function filterStateToWhereClause( + state: FilterState, + { + language, + escapeKey, + dateTimeColumns, + }: { + language: 'lucene' | 'sql'; + /** SQL only: map a clean key to its quoted ClickHouse expression. */ + escapeKey?: (key: string) => string; + /** SQL only: Map of DateTime/Date column name → its ClickHouse type. */ + dateTimeColumns?: ReadonlyMap; + }, +): string { + if (language === 'sql') { + const escaped: FilterState = {}; + for (const [key, values] of Object.entries(state)) { + escaped[escapeKey ? escapeKey(key) : key] = values; + } + return filtersToQuery(escaped, { dateTimeColumns }) + .flatMap(f => ('condition' in f ? [f.condition] : [])) + .join(' AND '); + } + + const clauses: string[] = []; + for (const [key, values] of Object.entries(state)) { + if ( + values.included.size === 0 && + values.excluded.size === 0 && + values.range == null + ) { + continue; + } + const luceneField = escapeLuceneFieldName(parseKeyPath(key).join('.')); + + if (values.included.size > 0) { + const terms = Array.from(values.included).map( + v => `${luceneField}:"${escapeLuceneQuotedTerm(String(v))}"`, + ); + clauses.push(terms.length > 1 ? `(${terms.join(' OR ')})` : terms[0]); + } + if (values.excluded.size > 0) { + clauses.push( + Array.from(values.excluded) + .map(v => `-${luceneField}:"${escapeLuceneQuotedTerm(String(v))}"`) + .join(' AND '), + ); + } + if (values.range != null) { + const open = + values.range.inclusive === 'none' || values.range.inclusive === 'right' + ? '{' + : '['; + const close = + values.range.inclusive === 'none' || values.range.inclusive === 'left' + ? '}' + : ']'; + clauses.push( + `${luceneField}:${open}${values.range.min} TO ${values.range.max}${close}`, + ); + } + } + return clauses.join(' AND '); +} + +type CollectedTerm = { field: string; value: string; negated: boolean }; +type CollectedRange = { + field: string; + min: number; + max: number; + inclusive?: 'both' | 'none' | 'left' | 'right'; +}; + +/** + * Return true when a `NodeTerm` carries a proximity (~N), fuzzy (~), or boost + * (^N) modifier. The `@hyperdx/lucene` package ships no `.d.ts` file so + * TypeScript infers these optional properties as absent from `NodeTerm`; we + * access them through an `unknown` cast to stay type-safe while still checking + * the runtime values the grammar always sets (to `null` when absent). + */ +function hasTermModifiers(node: lucene.NodeTerm): boolean { + const n = node as unknown as Record; + return ( + n['proximity'] != null || n['boost'] != null || n['similarity'] != null + ); +} + +/** + * Collect quoted terms and range terms from a Lucene AST into the `terms` / + * `ranges` arrays. `managedFields` collects every explicit-field name that we + * should treat as "managed" even when we cannot faithfully round-trip the value + * (unquoted terms, field groups). `collectFromAst` is only called in contexts + * that build the `managed` set, so this wider set lets `renderNode` prune + * unquoted field clauses (Bug 7) and field-group clauses (Bug 3). + * + * Cross-field OR rules (Bug 2): when an OR node connects clauses for *different* + * fields, the relationship cannot be expressed in FilterState (which has AND + * semantics across fields), so we treat the whole OR sub-tree as unmanaged — + * neither side is collected, and `renderNode` will preserve the raw text. + */ +function collectFromAst( + ast: lucene.AST | lucene.Node, + terms: CollectedTerm[], + ranges: CollectedRange[], + managedFields: Set, + negate = false, +): void { + if (isNodeTerm(ast)) { + // `negate` carries a `NOT` (or `+`/`-`) wrapper from an enclosing node, so + // `NOT a:"1"` / `a AND NOT b:"2"` collect as excluded instead of included. + // A `-` prefix XORs with the wrapper (`NOT -a` is not negated). Only a + // literal `-` prefix is sliced off the field name. + const negatedField = ast.field.startsWith('-'); + const negated = negatedField !== negate; + const field = decodeSpecialTokens( + negatedField ? ast.field.slice(1) : ast.field, + ); + // Implicit-field terms (free-text, quoted phrases) are never managed facets. + if (field === IMPLICIT_FIELD) return; + + // Terms with proximity (~N), fuzzy (~), or boost (^N) modifiers cannot be + // faithfully represented in FilterState, so treat them as unmanaged (Bug 6). + // However the field is still marked managed so that sibling plain clauses + // for the same field are pruned by renderNode rather than left to pile up + // alongside the new emission. + if (hasTermModifiers(ast)) { + managedFields.add(field); + return; + } + + // Collect the field as managed so renderNode can prune it (Bug 7). + managedFields.add(field); + + if (!ast.quoted) { + // Unquoted terms are managed for *removal* (so the old unquoted clause + // doesn't linger when the sidebar replaces it with a quoted one), but we + // don't add them to `terms` — the sidebar always emits quoted values. + return; + } + + terms.push({ field, value: decodeSpecialTokens(ast.term), negated }); + } else if (isNodeRangedTerm(ast)) { + const field = decodeSpecialTokens(ast.field); + if (field === IMPLICIT_FIELD) return; + // A negated range (`NOT duration:[10 TO 20]`) can't be represented in + // FilterState, so leave it unmanaged and preserve it verbatim. But still + // mark the field as managed so plain range clauses on the same field don't + // survive alongside the new emission. + if (negate) { + managedFields.add(field); + return; + } + const min = parseFloat(ast.term_min); + const max = parseFloat(ast.term_max); + if (!isNaN(min) && !isNaN(max)) { + managedFields.add(field); + const inclusive = + (ast.inclusive as 'both' | 'none' | 'left' | 'right') ?? 'both'; + ranges.push({ + field, + min, + max, + // Only store non-default inclusivity so round-tripping `[min TO max]` + // doesn't add an unexpected `inclusive` key to the range object. + ...(inclusive !== 'both' ? { inclusive } : {}), + }); + } + } else if (isBinaryAST(ast)) { + // Field-group syntax: ServiceName:("api" OR "web"). The parser puts a + // non-implicit `field` on the BinaryAST wrapper (Bug 3). Collect its + // inner leaf values as if they had the group's field name. + const groupField = + 'field' in ast && + typeof ast.field === 'string' && + ast.field !== IMPLICIT_FIELD + ? decodeSpecialTokens(ast.field) + : null; + + if (groupField) { + // Collect each leaf of the field group under the group's field name. + managedFields.add(groupField); + collectFieldGroupLeaves(ast, groupField, terms, managedFields, negate); + return; + } + + // Cross-field OR: if this OR node connects clauses for *different* fields, + // we cannot represent that in FilterState, so leave both sides unmanaged + // (Bug 2). Same-field OR (e.g. `level:"info" OR level:"warn"`) is fine — + // both leaves map to the same field and end up as multiple `included` values. + // `OR NOT` is an OR too (a:"1" OR NOT b:"2" is cross-field and unrepresentable). + if (ast.operator === 'OR' || ast.operator === 'OR NOT') { + const leftFields = new Set(); + const rightFields = new Set(); + const leftManaged = new Set(); + const rightManaged = new Set(); + collectFromAst(ast.left, [], [], leftFields); + collectFromAst(ast.right, [], [], rightFields); + // Merge managed fields from each side for the lookup + for (const f of leftFields) leftManaged.add(f); + for (const f of rightFields) rightManaged.add(f); + const isSameField = + leftFields.size > 0 && + rightFields.size > 0 && + leftFields.size === 1 && + rightFields.size === 1 && + [...leftFields][0] === [...rightFields][0]; + + if (!isSameField) { + // Cross-field OR — do not add either side to managed, leave raw. + return; + } + } + + // A `start: 'NOT'` on the binary node negates its left subtree (`NOT a AND + // b`); an `AND NOT` / `OR NOT` operator negates the right subtree. + const startOp: string | undefined = + 'start' in ast && typeof ast.start === 'string' ? ast.start : undefined; + const leftNegate = negate !== (startOp === 'NOT'); + const rightNegate = + negate !== (ast.operator === 'AND NOT' || ast.operator === 'OR NOT'); + collectFromAst(ast.left, terms, ranges, managedFields, leftNegate); + collectFromAst(ast.right, terms, ranges, managedFields, rightNegate); + } else if (isLeftOnlyAST(ast)) { + collectFromAst( + ast.left, + terms, + ranges, + managedFields, + negate !== (ast.start === 'NOT'), + ); + } +} + +/** + * Recursively collect leaf terms from a field-group BinaryAST, assigning them + * the `groupField` instead of ``. + */ +function collectFieldGroupLeaves( + ast: lucene.AST | lucene.Node, + groupField: string, + terms: CollectedTerm[], + managedFields: Set, + negate = false, +): void { + if (isNodeTerm(ast)) { + // Skip modifiers — proximity/boost can't be faithfully round-tripped. + if (hasTermModifiers(ast)) { + return; + } + if (!ast.quoted) return; // unquoted inside group: add field but no value + const negated = ast.field.startsWith('-') !== negate; + terms.push({ + field: groupField, + value: decodeSpecialTokens(ast.term), + negated, + }); + } else if (isBinaryAST(ast)) { + collectFieldGroupLeaves(ast.left, groupField, terms, managedFields, negate); + collectFieldGroupLeaves( + ast.right, + groupField, + terms, + managedFields, + negate, + ); + } else if (isLeftOnlyAST(ast)) { + collectFieldGroupLeaves(ast.left, groupField, terms, managedFields, negate); + } +} + +/** Coerce "true"/"false" strings back to booleans, pass through otherwise */ +export function coerceBooleanValue(v: string | boolean): string | boolean { + if (typeof v === 'boolean') return v; + if (v === 'true') return true; + if (v === 'false') return false; + return v; +} + +/** + * Parse a `where` clause string back into a FilterState (clean keys), the + * inverse of `filterStateToWhereClause`. + * + * - `sql`: delegates to `parseQuery`, so keys come back in their raw SQL + * (quoted/bracket) form — callers that need clean keys unescape them. + * - `lucene`: collects quoted `field:"value"` terms (included / negated = + * excluded) and `field:[min TO max]` ranges. Only clauses matching the + * facet grammar round-trip; free-text and complex queries are ignored. + * + * Returns an empty state when the text is empty or (for lucene) fails to parse. + */ +export function parseWhereClauseToFilterState( + whereText: string, + language: 'lucene' | 'sql', +): FilterState { + if (language === 'sql') { + return parseQuery([{ type: 'sql', condition: whereText }]).filters; + } + + try { + const ast = parse(whereText); + const terms: CollectedTerm[] = []; + const ranges: CollectedRange[] = []; + const managedFields = new Set(); + collectFromAst(ast, terms, ranges, managedFields); + + const byField = new Map< + string, + { + included: Set; + excluded: Set; + range?: { + min: number; + max: number; + inclusive?: 'both' | 'none' | 'left' | 'right'; + }; + } + >(); + const getEntry = (field: string) => { + if (!byField.has(field)) { + byField.set(field, { included: new Set(), excluded: new Set() }); + } + return byField.get(field)!; + }; + + for (const t of terms) { + const entry = getEntry(t.field); + const value = coerceBooleanValue(t.value); + if (t.negated) { + entry.excluded.add(value); + } else { + entry.included.add(value); + } + } + for (const r of ranges) { + const entry = getEntry(r.field); + entry.range = { + min: r.min, + max: r.max, + ...(r.inclusive != null ? { inclusive: r.inclusive } : {}), + }; + } + return Object.fromEntries(byField); + } catch { + return {}; + } +} + +/** + * Return a non-null message when a `where` clause cannot be parsed in the given + * query language (e.g. an incomplete lucene query like `service:`). The message + * is intended for a UI notice — callers can treat any non-null value as "the + * query is invalid/incomplete". + * + * `sql` always returns null: there is no strict SQL parser here and + * `parseQuery` is tolerant, so we can't reliably distinguish invalid from valid. + */ +export function getWhereParseError( + whereText: string, + language: 'lucene' | 'sql', +): string | null { + if (!whereText.trim()) return null; + if (language === 'sql') return null; + try { + parse(whereText); + return null; + } catch (e) { + return e instanceof Error ? e.message : 'Invalid query'; + } +} + +/** + * Collect every explicit-field name referenced under a lucene AST node, using + * the field's decoded name (stripping `-`/`+` prefixes). Used to detect + * cross-field OR constructs. + */ +function collectExplicitFields( + node: lucene.AST | lucene.Node, + out: Set, +): void { + if (isNodeTerm(node)) { + const raw = + node.field.startsWith('-') || node.field.startsWith('+') + ? node.field.slice(1) + : node.field; + const field = decodeSpecialTokens(raw); + if (field !== IMPLICIT_FIELD) out.add(field); + return; + } + if (isNodeRangedTerm(node)) { + const field = decodeSpecialTokens(node.field); + if (field !== IMPLICIT_FIELD) out.add(field); + return; + } + if (isBinaryAST(node)) { + const groupField = + 'field' in node && + typeof node.field === 'string' && + node.field !== IMPLICIT_FIELD + ? decodeSpecialTokens(node.field) + : null; + if (groupField) { + // Field-group syntax: all inner leaves belong to the group's field. + out.add(groupField); + return; + } + collectExplicitFields(node.left, out); + collectExplicitFields(node.right, out); + return; + } + if (isLeftOnlyAST(node)) { + collectExplicitFields(node.left, out); + } +} + +/** + * Return true when a lucene AST contains an OR (or OR NOT) node connecting + * clauses for different explicit fields. Such a query cannot be represented as + * a FilterState (which has AND semantics across fields), so the sidebar would + * either silently drop it or mislead. + */ +function hasCrossFieldOr(node: lucene.AST | lucene.Node): boolean { + if (isNodeTerm(node) || isNodeRangedTerm(node)) return false; + if (isLeftOnlyAST(node)) return hasCrossFieldOr(node.left); + if (isBinaryAST(node)) { + const groupField = + 'field' in node && + typeof node.field === 'string' && + node.field !== IMPLICIT_FIELD + ? decodeSpecialTokens(node.field) + : null; + if (groupField) { + return hasCrossFieldOr(node.left) || hasCrossFieldOr(node.right); + } + if (node.operator === 'OR' || node.operator === 'OR NOT') { + const leftFields = new Set(); + const rightFields = new Set(); + collectExplicitFields(node.left, leftFields); + collectExplicitFields(node.right, rightFields); + const isSameField = + leftFields.size > 0 && + rightFields.size > 0 && + leftFields.size === 1 && + rightFields.size === 1 && + [...leftFields][0] === [...rightFields][0]; + if (!isSameField) return true; + } + return hasCrossFieldOr(node.left) || hasCrossFieldOr(node.right); + } + return false; +} + +/** + * Return a non-null reason when a `where` clause parses but contains facet-like + * content the sidebar FilterState cannot faithfully represent — currently a + * cross-field OR (e.g. `ServiceName:"api" OR SeverityText:"error"`). The UI + * should surface this instead of showing a misleading or silently-emptied + * filter sidebar. `sql` always returns null. + */ +export function getUnrepresentableWhereReason( + whereText: string, + language: 'lucene' | 'sql', +): string | null { + if (!whereText.trim()) return null; + if (language === 'sql') return null; + try { + const ast = parse(whereText); + if (hasCrossFieldOr(ast)) { + return 'This query contains OR conditions between different fields, which cannot be shown as sidebar filters.'; + } + return null; + } catch { + return null; + } +} + +type Span = { start: number; end: number }; + +function termClauseSpan(node: lucene.NodeTerm, src: string): Span | null { + const tl = node.termLocation; + if (!tl?.end) return null; + const start = node.fieldLocation?.start?.offset ?? tl.start?.offset; + if (start == null) return null; + // Bug 10 fix: termLocation.end.offset can include trailing whitespace (the + // grammar's `_*` consumes whitespace after the term). Trim to avoid + // accumulating extra spaces when the residual is re-joined with an operator. + let end = tl.end.offset; + while (end > start && (src[end - 1] === ' ' || src[end - 1] === '\t')) { + end--; + } + return { start, end }; +} + +function rangeClauseSpan( + node: lucene.NodeRangedTerm, + src: string, +): Span | null { + const fl = node.fieldLocation; + // Bug 4 fix: use == null instead of falsy check so offset=0 is not dropped. + if (fl?.start?.offset == null || !fl.end?.offset) return null; + // Bug 5 fix: the range may end with `]` (inclusive) or `}` (exclusive). + // Search for whichever closing bracket appears first after the field end. + const searchFrom = fl.end.offset; + const closeSq = src.indexOf(']', searchFrom); + const closeCurly = src.indexOf('}', searchFrom); + let endBracket: number; + if (closeSq === -1 && closeCurly === -1) return null; + if (closeSq === -1) endBracket = closeCurly; + else if (closeCurly === -1) endBracket = closeSq; + else endBracket = Math.min(closeSq, closeCurly); + return { start: fl.start.offset, end: endBracket + 1 }; +} + +type RenderResult = { + /** Rendered residual text (empty when every leaf was a facet clause). */ + text: string; + /** Whether every leaf under this node is a managed facet clause. */ + fullyManaged: boolean; +}; + +/** + * Render a Lucene AST back to text, pruning every leaf whose field is in + * `isManagedField` and keeping the raw source text of everything else. Sibling + * connectors are re-emitted from the tree so that removing a clause from an + * OR/AND chain doesn't leave dangling operators (e.g. removing `host` from + * `foo:"x" AND host:"a" AND bar:"y"` yields `foo:"x" AND bar:"y"`, and from + * `(a OR host OR b)` yields `(a OR b)`). + * + * `negate` mirrors the same flag in `collectFromAst`: negated ranges + * (`NOT duration:[...]`) cannot be represented in FilterState and must be + * preserved verbatim even when their field is managed. + */ +function renderNode( + node: lucene.AST | lucene.Node, + src: string, + isManagedField: (field: string) => boolean, + negate = false, +): RenderResult { + if (isNodeTerm(node)) { + const span = termClauseSpan(node, src); + if (!span) return { text: '', fullyManaged: false }; + if (!node.quoted) { + // Bug 7 fix: unquoted explicit-field terms (e.g. level:error) must also + // be pruned when their field is managed, so the sidebar can replace them + // with a quoted clause without duplication. + const rawField = node.field.startsWith('-') + ? node.field.slice(1) + : node.field; + const field = decodeSpecialTokens(rawField); + if (field !== IMPLICIT_FIELD && isManagedField(field)) { + return { text: '', fullyManaged: true }; + } + return { text: src.slice(span.start, span.end), fullyManaged: false }; + } + const field = decodeSpecialTokens( + node.field.startsWith('-') ? node.field.slice(1) : node.field, + ); + // Terms with modifiers (proximity, boost, fuzzy) are not managed. + if (hasTermModifiers(node)) { + return { text: src.slice(span.start, span.end), fullyManaged: false }; + } + if (field !== IMPLICIT_FIELD && isManagedField(field)) { + return { text: '', fullyManaged: true }; + } + return { text: src.slice(span.start, span.end), fullyManaged: false }; + } + + if (isNodeRangedTerm(node)) { + const span = rangeClauseSpan(node, src); + if (!span) return { text: '', fullyManaged: false }; + const field = decodeSpecialTokens(node.field); + // Negated ranges (NOT duration:[...]) are not representable in FilterState + // and must be preserved verbatim even when the field is otherwise managed. + if (!negate && field !== IMPLICIT_FIELD && isManagedField(field)) { + return { text: '', fullyManaged: true }; + } + return { text: src.slice(span.start, span.end), fullyManaged: false }; + } + + if (isLeftOnlyAST(node)) { + const nodeNegate = negate !== (node.start === 'NOT'); + const inner = renderNode(node.left, src, isManagedField, nodeNegate); + if (inner.fullyManaged) { + return { text: '', fullyManaged: true }; + } + const prefix = node.start ? `${node.start} ` : ''; + return { + text: inner.text ? `${prefix}${inner.text}` : '', + fullyManaged: false, + }; + } + + if (isBinaryAST(node)) { + // Field-group syntax: ServiceName:("api" OR "web"). The parser attaches + // a non-implicit `field` to the BinaryAST wrapper (Bug 3). When the group + // field is managed we discard the whole group; otherwise we preserve the + // raw source including the "Field:" prefix. + const groupField = + 'field' in node && + typeof node.field === 'string' && + node.field !== IMPLICIT_FIELD + ? decodeSpecialTokens(node.field) + : null; + + if (groupField) { + if (isManagedField(groupField)) { + return { text: '', fullyManaged: true }; + } + // Preserve raw source. The field-group span starts at the field's + // fieldLocation and ends after the closing ')'. + const fl = + 'fieldLocation' in node + ? (node.fieldLocation as + | { start?: { offset?: number }; end?: { offset?: number } } + | null + | undefined) + : undefined; + const groupStart = fl?.start?.offset; + if (groupStart == null) return { text: '', fullyManaged: false }; + // Find the closing ')' of the paren group. + let depth = 0; + let groupEnd = -1; + for (let i = groupStart; i < src.length; i++) { + if (src[i] === '(') depth++; + else if (src[i] === ')') { + depth--; + if (depth === 0) { + groupEnd = i + 1; + break; + } + } + } + if (groupEnd === -1) return { text: '', fullyManaged: false }; + return { + text: src.slice(groupStart, groupEnd).trimEnd(), + fullyManaged: false, + }; + } + + const startOp: string | undefined = + 'start' in node && typeof node.start === 'string' ? node.start : undefined; + const leftNegate = negate !== (startOp === 'NOT'); + const rightNegate = + negate !== (node.operator === 'AND NOT' || node.operator === 'OR NOT'); + const left = renderNode(node.left, src, isManagedField, leftNegate); + const right = renderNode(node.right, src, isManagedField, rightNegate); + if (left.fullyManaged && right.fullyManaged) { + return { text: '', fullyManaged: true }; + } + const operator = + node.operator === '' ? ' ' : ` ${node.operator} `; + const combined = [left.text, right.text].filter(Boolean).join(operator); + // Bug 1 fix: the `start` field carries a leading operator like `NOT` that + // the grammar placed on the outer binary node rather than a LeftOnlyAST. + // Re-emit it so `NOT term AND ...` is not silently stripped to `term AND ...`. + const startPrefix = 'start' in node && node.start ? `${node.start} ` : ''; + const text = node.parenthesized && combined ? `(${combined})` : combined; + return { + text: startPrefix && text ? `${startPrefix}${text}` : text, + fullyManaged: false, + }; + } + + return { text: '', fullyManaged: false }; +} + +/** + * Returns true when the Lucene text contains a top-level OR operator (i.e. an + * OR that is not inside parentheses or quoted strings). Used to decide whether + * the residual must be wrapped in parens before joining it with AND — without + * the wrapping, `a OR b AND c:"v"` would be mis-parsed as `a OR (b AND c:"v")`. + * + * Bug 9 fix: the original implementation counted bare `(` / `)` characters + * without skipping quoted strings, so a `)` inside a quoted value like + * `"timeout)"` decremented the paren depth and corrupted the subsequent scan. + */ +function hasTopLevelOr(text: string): boolean { + let parenDepth = 0; + let inQuote = false; + for (let i = 0; i < text.length; i++) { + const ch = text[i]; + if (ch === '\\' && inQuote) { + // Skip escaped character inside a quoted string. + i++; + continue; + } + if (ch === '"') { + inQuote = !inQuote; + continue; + } + if (inQuote) continue; + if (ch === '(') { + parenDepth++; + continue; + } + if (ch === ')') { + parenDepth--; + continue; + } + if (parenDepth === 0 && text.slice(i, i + 4).toUpperCase() === ' OR ') { + return true; + } + } + return false; +} + +/** + * Returns true when the SQL text contains a top-level OR operator (i.e. an OR + * that is not inside parentheses, single-quoted strings, or backtick-quoted + * identifiers). Used to decide whether the residual must be wrapped in parens + * before joining it with AND — without the wrapping, `a = 1 OR b = 2 AND c IN + * ('v')` would be mis-parsed as `a = 1 OR (b = 2 AND c IN ('v'))`. + */ +function hasTopLevelOrSql(text: string): boolean { + let parenDepth = 0; + let inString = false; + let inBacktick = false; + for (let i = 0; i < text.length; i++) { + const ch = text[i]; + if (inString) { + if (ch === "'") { + const esc = handleQuoteEscape(text, i); + if (esc.skip) { + i = esc.next; + continue; + } + inString = false; + } + continue; + } + if (inBacktick) { + if (ch === '`') inBacktick = false; + continue; + } + if (ch === "'") { + inString = true; + continue; + } + if (ch === '`') { + inBacktick = true; + continue; + } + if (ch === '(') { + parenDepth++; + continue; + } + if (ch === ')') { + parenDepth--; + continue; + } + if (parenDepth === 0 && text.slice(i, i + 4).toUpperCase() === ' OR ') { + return true; + } + } + return false; +} + +/** + * Replace the facet clauses in a Lucene `where` clause with a new FilterState. + * + * Only clauses matching the facet grammar (quoted `field:"v"`, `-field:"v"`, + * `field:[a TO b]`) are touched. Free-text and unrecognized query content is + * preserved from the original text, with connectors rebuilt so no dangling + * `AND`/`OR`/parens are left behind. The new clauses are appended after the + * residual text. + */ +function replaceLuceneFacetClauses( + whereText: string, + newState: FilterState, + emitLanguage: 'lucene' | 'sql' | undefined, + escapeKey: ((key: string) => string) | undefined, + dateTimeColumns: ReadonlyMap | undefined, +): string { + if (!whereText.trim()) { + return filterStateToWhereClause(newState, { + language: emitLanguage ?? 'lucene', + escapeKey, + dateTimeColumns, + }); + } + + let ast: lucene.AST; + try { + ast = parse(whereText); + } catch { + // Invalid Lucene — the query text is incomplete/mid-edit. Rather than + // silently no-oping the caller's rewrite, emit the new state fresh so a + // sidebar click still applies and replaces the broken text. + return filterStateToWhereClause(newState, { + language: emitLanguage ?? 'lucene', + escapeKey, + dateTimeColumns, + }); + } + + // Managed fields are every facet field currently present in the text, so a + // caller that intentionally drops a field (e.g. source-change cleanup) also + // removes its clauses rather than leaving them to resurface. + // We use the wider `managedFields` set (which includes unquoted explicit-field + // terms and field-group fields, but excludes cross-field OR branches) so that + // e.g. an unquoted `level:error` is removed when the sidebar sets level:"warn". + const managedFields = new Set(); + collectFromAst(parse(whereText), [], [], managedFields); + const residual = renderNode(ast, whereText, field => + managedFields.has(field), + ).text; + const newClauses = filterStateToWhereClause(newState, { + language: emitLanguage ?? 'lucene', + escapeKey, + dateTimeColumns, + }); + + const trimmedResidual = residual.trim(); + if (!trimmedResidual) return newClauses; + if (!newClauses) return trimmedResidual; + // If the residual contains a top-level OR operator it must be parenthesized + // before joining with AND, otherwise `a OR b AND c:"v"` would be parsed as + // `a OR (b AND c:"v")` — narrowing the OR branch incorrectly. + const safeResidual = hasTopLevelOr(residual) + ? `(${trimmedResidual})` + : trimmedResidual; + return `${safeResidual} AND ${newClauses}`; +} + +/** + * Split a SQL `where` string into top-level conjuncts, splitting on ` AND ` + * outside quotes and parentheses. The ` AND ` that belongs to a `BETWEEN ... + * AND ...` range is not a separator, so a BETWEEN conjunct stays intact. + */ +function splitSqlConjuncts(text: string): string[] { + const conjuncts: string[] = []; + let current = ''; + let inString = false; + let inBacktick = false; + let parenDepth = 0; + let sawBetween = false; + + for (let i = 0; i < text.length; i++) { + const char = text[i]; + + // Backtick-quoted identifiers (e.g. `it's`) are opaque: a `'` or `(` inside + // them is literal and must not toggle string state or paren depth. + if (inBacktick) { + if (char === '`') inBacktick = false; + current += char; + continue; + } + + if (isQuoteBoundary(text, i)) { + if (inString) { + const esc = handleQuoteEscape(text, i); + if (esc.skip) { + current += "''"; + i = esc.next; + continue; + } + } + inString = !inString; + current += char; + continue; + } + + if (inString) { + current += char; + continue; + } + + if (char === '`') { + inBacktick = true; + current += char; + continue; + } + + if (char === '(') { + parenDepth++; + current += char; + continue; + } + if (char === ')') { + parenDepth--; + current += char; + continue; + } + + if (parenDepth > 0) { + current += char; + continue; + } + + if (!sawBetween && text.slice(i, i + 8).toUpperCase() === 'BETWEEN ') { + sawBetween = true; + current += 'BETWEEN '; + i += 7; + continue; + } + + if (text.slice(i, i + 5).toUpperCase() === ' AND ') { + if (sawBetween) { + // The range's own `AND` — not a conjunct separator. + sawBetween = false; + current += ' AND '; + i += 4; + continue; + } + if (current.trim()) { + conjuncts.push(current.trim()); + } + current = ''; + i += 4; + continue; + } + + current += char; + } + + if (current.trim()) { + conjuncts.push(current.trim()); + } + return conjuncts; +} + +// Strip SQL comments (`-- ...` line comments and `/* ... */` block comments) +// from a WHERE string, returning the code with each comment replaced by a single +// space plus the collected comment texts. Comments inside single-quoted strings +// or backtick-quoted identifiers are literal text and are left untouched. +// Stripping comments before conjunct splitting keeps a commented-out ` AND ` +// from being treated as a real separator, and lets the caller re-append the +// comments at the end instead of letting a facet predicate land inside one. +function stripSqlComments(text: string): { + code: string; + comments: string[]; +} { + let code = ''; + const comments: string[] = []; + let inString = false; + let inBacktick = false; + + for (let i = 0; i < text.length; i++) { + if (inBacktick) { + code += text[i]; + if (text[i] === '`') inBacktick = false; + continue; + } + if (inString) { + if (isQuoteBoundary(text, i)) { + const esc = handleQuoteEscape(text, i); + if (esc.skip) { + code += "''"; + i = esc.next; + continue; + } + inString = false; + } + code += text[i]; + continue; + } + if (text[i] === "'") { + inString = true; + code += text[i]; + continue; + } + if (text[i] === '`') { + inBacktick = true; + code += text[i]; + continue; + } + if (text[i] === '-' && text[i + 1] === '-') { + let comment = '--'; + i += 2; + while (i < text.length && text[i] !== '\n') { + comment += text[i]; + i++; + } + comments.push(comment); + code += ' '; + i--; // the \n (or end) will be consumed by the loop's own increment + continue; + } + if (text[i] === '/' && text[i + 1] === '*') { + let comment = '/*'; + i += 2; + while (i < text.length) { + if (text[i] === '*' && text[i + 1] === '/') { + comment += '*/'; + i += 2; + break; + } + comment += text[i]; + i++; + } + comments.push(comment); + code += ' '; + i--; // let the loop's increment advance past the closing '/' (or end) + continue; + } + code += text[i]; + } + + return { code, comments }; +} + +/** + * Replace the facet clauses in a SQL `where` clause with a new FilterState. + * + * Top-level conjuncts that parse as a single facet predicate (` IN + * (...)`, ` NOT IN (...)`, ` BETWEEN ... AND ...`) for a field being + * written are dropped and re-emitted from `newState`; everything else is + * preserved. `escapeKey` maps clean keys to the quoted ClickHouse expressions + * used both to match existing conjuncts and to emit new ones. + */ +function replaceSqlFacetClauses( + whereText: string, + newState: FilterState, + escapeKey: ((key: string) => string) | undefined, + dateTimeColumns: ReadonlyMap | undefined, + emitLanguage?: 'lucene' | 'sql', +): string { + const clean = whereText.trim(); + // Strip SQL comments up front so a commented-out ` AND ` is not mistaken for + // a real conjunct separator, and re-append them at the end so newly emitted + // facet predicates never land inside a comment. + const { code, comments } = stripSqlComments(clean); + const commentSuffix = comments.length ? ` ${comments.join(' ')}` : ''; + const emit = () => + filterStateToWhereClause(newState, { + language: emitLanguage ?? 'sql', + escapeKey, + dateTimeColumns, + }); + + if (!code.trim()) return `${emit()}${commentSuffix}`; + + const kept: string[] = []; + // Build the set of clean keys being managed (present in the original text as + // parseable facets) so we drop ALL of them and re-emit only those in newState. + // This lets the caller clear a field from the filter by omitting it from + // newState, and also handles the case where newState is empty (clear all). + // + // A key is only managed when it appears *exactly once* as a facet conjunct. + // If the same key appears in multiple conjuncts (e.g. `host IN ('a') AND host + // IN ('b')`) the user intentionally wrote a conjunction of two IN lists. + // Merging them into one IN list would change the semantics (intersection → + // union for scalar columns), so we leave both conjuncts untouched instead. + const keyCount = new Map(); + for (const conjunct of splitSqlConjuncts(code)) { + if (!conjunct.trim()) continue; + const upper = conjunct.toUpperCase(); + const isFacet = + upper.includes(' IN (') || + upper.includes(' NOT IN (') || + upper.includes(' BETWEEN '); + if (!isFacet) continue; + const filter: Filter = { type: 'sql', condition: conjunct }; + if (isRenderablePinnedFilter(filter)) { + const parsedKey = Object.keys(parseQuery([filter]).filters)[0]; + if (parsedKey !== undefined) { + keyCount.set(parsedKey, (keyCount.get(parsedKey) ?? 0) + 1); + } + } + } + // Only keys that appear exactly once are safe to replace. + const managedKeys = new Set( + [...keyCount.entries()] + .filter(([, count]) => count === 1) + .map(([key]) => key), + ); + + for (const conjunct of splitSqlConjuncts(code)) { + if (!conjunct.trim()) continue; + const upper = conjunct.toUpperCase(); + const isFacet = + upper.includes(' IN (') || + upper.includes(' NOT IN (') || + upper.includes(' BETWEEN '); + if (!isFacet) { + kept.push(conjunct); + continue; + } + const filter: Filter = { type: 'sql', condition: conjunct }; + // Drop this conjunct if it maps to a managed field — it will be re-emitted + // from newState (or omitted entirely if newState doesn't include that field). + if (isRenderablePinnedFilter(filter)) { + const parsedKey = Object.keys(parseQuery([filter]).filters)[0]; + if (parsedKey !== undefined && managedKeys.has(parsedKey)) { + continue; // will be re-emitted from newState (or dropped if not there) + } + } + kept.push(conjunct); + } + + const residual = kept.join(' AND '); + const newClauses = emit(); + if (!residual) return `${newClauses}${commentSuffix}`; + if (!newClauses) return `${residual}${commentSuffix}`; + // If the residual contains a top-level OR operator it must be parenthesized + // before joining with AND, otherwise `a = 1 OR b = 2 AND c IN ('v')` would be + // parsed as `a = 1 OR (b = 2 AND c IN ('v'))` — narrowing the OR branch. + const safeResidual = hasTopLevelOrSql(residual) + ? `(${residual.trim()})` + : residual; + return `${safeResidual} AND ${newClauses}${commentSuffix}`; +} + +/** + * Replace the facet clauses in a `where` clause with a new FilterState, in the + * given query language. Non-facet content (free-text, complex queries) is + * preserved; only clauses matching the facet grammar are rewritten. + * + * This is the write half of the search page's unified sidebar-⇄-query-input + * representation: sidebar toggles produce a FilterState, which replaces the + * matching clauses in the query text instead of living in a separate `filters` + * parameter. + */ +export function replaceFilterClauses( + whereText: string, + language: 'lucene' | 'sql', + newState: FilterState, + { + escapeKey, + dateTimeColumns, + emitLanguage, + }: { + escapeKey?: (key: string) => string; + dateTimeColumns?: ReadonlyMap; + /** + * When set, the facets are re-emitted in this language instead of `language`. + * Matching/pruning still happens in `language` (the source of the existing + * text); `emitLanguage` only changes how the new clauses are written. Used + * to translate a `where` clause from one query language to another while + * preserving non-facet content. + */ + emitLanguage?: 'lucene' | 'sql'; + } = {}, +): string { + if (language === 'lucene') { + return replaceLuceneFacetClauses( + whereText, + newState, + emitLanguage, + escapeKey, + dateTimeColumns, + ); + } + return replaceSqlFacetClauses( + whereText, + newState, + escapeKey, + dateTimeColumns, + emitLanguage, + ); +} + +/** + * Append `newState`'s clauses to a `where` clause while preserving the existing + * text verbatim. Unlike `replaceFilterClauses` (which treats the existing text's + * facet fields as managed and re-emits only the new state's fields), this *merges*: + * both the original clause and the new state's predicates are kept, joined with + * AND. + * + * This is the semantics a legacy `where` + separate `filters` representation + * needs when folding the filters into the unified `where`: the two were + * independent predicates ANDed at query time, so neither may be dropped. + * + * The existing text is preserved verbatim (it is not re-emitted), so complex or + * free-form content round-trips exactly. Only two adjustments are made to keep + * the joined clause well-formed: + * - a top-level OR in the existing text is parenthesized before the AND join, + * so `a OR b AND c:"v"` is not mis-parsed as `a OR (b AND c:"v")`; + * - SQL comments in the existing text are moved to the end so the appended + * predicate does not land inside a `--` / `/* */ +export function mergeFilterStateIntoWhereClause( + whereText: string, + language: 'lucene' | 'sql', + newState: FilterState, + { + escapeKey, + dateTimeColumns, + }: { + escapeKey?: (key: string) => string; + dateTimeColumns?: ReadonlyMap; + } = {}, +): string { + const newClauses = filterStateToWhereClause(newState, { + language, + escapeKey, + dateTimeColumns, + }); + if (!whereText.trim()) return newClauses; + if (!newClauses) return whereText.trim(); + + if (language === 'sql') { + const { code, comments } = stripSqlComments(whereText.trim()); + const residual = code.trim(); + const safeResidual = hasTopLevelOrSql(residual) + ? `(${residual})` + : residual; + const commentSuffix = comments.length ? ` ${comments.join(' ')}` : ''; + return `${safeResidual} AND ${newClauses}${commentSuffix}`; + } + + const safeWhere = hasTopLevelOr(whereText) + ? `(${whereText.trim()})` + : whereText.trim(); + return `${safeWhere} AND ${newClauses}`; +} + // Helper function to parse a string value as boolean if possible, or otherwise // return as string with surrounding quotes removed and SQL-escaped quotes unescaped. const getBooleanOrUnquotedString = (value: string): string | boolean => { @@ -388,6 +1642,29 @@ function extractInClauses(condition: string): Array<{ const keyStr = key.trim(); const trimmedValues = values.trim(); + + // A subquery (`col IN (SELECT ...)`) is not a facet: treating its inner + // SQL as a plain value list would both render a bogus checkbox in the + // sidebar and let `replaceSqlFacetClauses` destroy the subquery when + // re-emitting the column. Reject any parenthesized list containing SQL + // clause keywords outside quoted strings. + if ( + containsOutsideQuotes(trimmedValues, [ + ' SELECT ', + ' FROM ', + ' WHERE ', + ' GROUP ', + ' ORDER ', + ' UNION ', + ' HAVING ', + ' JOIN ', + ' LIMIT ', + ' DISTINCT ', + ]) + ) { + continue; + } + const withoutParens = trimmedValues.startsWith('(') && trimmedValues.endsWith(')') ? trimmedValues.slice(1, -1) diff --git a/packages/common-utils/src/queryParser.ts b/packages/common-utils/src/queryParser.ts index 0c5a2621d8..cababc8da2 100644 --- a/packages/common-utils/src/queryParser.ts +++ b/packages/common-utils/src/queryParser.ts @@ -14,6 +14,7 @@ import { isClickHouseVersionAtLeast, supportsDirectReadMap, } from '@/core/clickhouseVersion'; +import { dateTimeValueExpr } from '@/core/dateTimeValue'; import { Metadata, parseKeyPath, @@ -37,7 +38,7 @@ function encodeSpecialTokens(query: string): string { .replace(/localhost:(\d{1,5})/, 'localhost_COLON_$1') .replace(/\\:/g, 'HDX_COLON'); } -function decodeSpecialTokens(query: string): string { +export function decodeSpecialTokens(query: string): string { return query .replace(/\\"/g, '"') .replace(/HDX_BACKSLASH_LITERAL/g, '\\') @@ -65,17 +66,21 @@ function normalizeChExpression(expr: string): string { const IMPLICIT_FIELD = ''; // Type guards for lucene AST types -function isNodeTerm(node: lucene.Node | lucene.AST): node is lucene.NodeTerm { +export function isNodeTerm( + node: lucene.Node | lucene.AST, +): node is lucene.NodeTerm { return 'term' in node && node.term != null; } -function isNodeRangedTerm( +export function isNodeRangedTerm( node: lucene.Node | lucene.AST, ): node is lucene.NodeRangedTerm { return 'inclusive' in node && node.inclusive != null; } -function isBinaryAST(ast: lucene.AST | lucene.Node): ast is lucene.BinaryAST { +export function isBinaryAST( + ast: lucene.AST | lucene.Node, +): ast is lucene.BinaryAST { return 'right' in ast && ast.right != null; } @@ -85,7 +90,7 @@ function hasStart( return 'start' in ast && !!ast.start; } -function isLeftOnlyAST( +export function isLeftOnlyAST( ast: lucene.AST | lucene.Node, ): ast is lucene.LeftOnlyAST { return ( @@ -407,6 +412,7 @@ export abstract class SQLSerializer implements Serializer { ): Promise<{ column?: string; columnJSON?: { string: string; number: string }; + columnType?: string; propertyType?: JSDataType; isArray?: boolean; found: boolean; @@ -446,6 +452,7 @@ export abstract class SQLSerializer implements Serializer { const { column, columnJSON, + columnType, found, propertyType, isArray, @@ -495,6 +502,19 @@ export abstract class SQLSerializer implements Serializer { mapKeyIndexExpression && !isNegatedField ? ` AND ${mapKeyIndexExpression}` : ''; + + // DateTime / DateTime64 / Date columns can't be compared against a bare + // string literal in ClickHouse, so wrap the value in the same parse/convert + // expression the SQL filter emitter uses (`dateTimeValueExpr`). Without + // this, an exact-match quoted term (e.g. `Timestamp:"2026-06-16T...Z"`) + // renders `Timestamp = '2026-06-16T...Z'`, which ClickHouse rejects. + if (columnType != null && /\b(?:DateTime64?|Date32?)\b/.test(columnType)) { + return SqlString.format( + `(${column} ${isNegatedField ? '!' : ''}= ?${expressionPostfix})`, + [SqlString.raw(dateTimeValueExpr(columnType, SqlString.escape(term)))], + ); + } + if (propertyType === JSDataType.Bool) { // numeric and boolean fields must be equality matched const normTerm = `${term}`.trim().toLowerCase(); @@ -713,7 +733,7 @@ export abstract class SQLSerializer implements Serializer { isNegatedField: boolean, context: SerializerContext, ) { - const { column, found, mapKeyIndexExpression, isArray } = + const { column, columnType, found, mapKeyIndexExpression, isArray } = await this.getColumnForField(field, context); if (!found) { return this.NOT_FOUND_QUERY; @@ -727,6 +747,17 @@ export abstract class SQLSerializer implements Serializer { mapKeyIndexExpression && !isNegatedField ? ` AND ${mapKeyIndexExpression}` : ''; + + // Date columns need parse/convert-wrapped bounds, mirroring dateTimeValueExpr. + if (columnType != null && /\b(?:DateTime64?|Date32?)\b/.test(columnType)) { + return SqlString.format( + `(${column} ${isNegatedField ? 'NOT ' : ''}BETWEEN ? AND ?${expressionPostfix})`, + [ + SqlString.raw(dateTimeValueExpr(columnType, SqlString.escape(start))), + SqlString.raw(dateTimeValueExpr(columnType, SqlString.escape(end))), + ], + ); + } return SqlString.format( `(${column} ${isNegatedField ? 'NOT ' : ''}BETWEEN ? AND ?${expressionPostfix})`, [this.attemptToParseNumber(start), this.attemptToParseNumber(end)], @@ -1868,6 +1899,7 @@ export class CustomSchemaSQLSerializerV2 extends SQLSerializer { return { column: expression.columnExpression, columnJSON: expression?.columnExpressionJSON, + columnType: expression.columnType, propertyType: type ?? undefined, isArray, found: expression.found, @@ -1888,9 +1920,14 @@ async function nodeTerm( serializer: Serializer, context: SerializerContext, ): Promise { - const field = node.field[0] === '-' ? node.field.slice(1) : node.field; - let isNegatedField = node.field[0] === '-'; const isImplicitField = node.field === IMPLICIT_FIELD; + const rawField = node.field[0] === '-' ? node.field.slice(1) : node.field; + // Decode special-token placeholders the emitter inserted in the field name + // (e.g. HDX_COLON for an escaped `:` in a Map sub-key). Leave the implicit + // field sentinel untouched so downstream comparisons against IMPLICIT_FIELD + // still match. + const field = isImplicitField ? rawField : decodeSpecialTokens(rawField); + let isNegatedField = node.field[0] === '-'; // NodeTerm if (isNodeTerm(node)) { @@ -1992,7 +2029,7 @@ function createSerializerContext( return { ...currentContext, - implicitColumnExpression: fieldWithoutNegation, + implicitColumnExpression: decodeSpecialTokens(fieldWithoutNegation), ...(isNegatedAndParenthesized(ast) ? { isNegatedAndParenthesized: true } : {}),