feat: unify filter sidebar and query input into single where clause

.changeset/quiet-lions-repeat.md

@@ -0,0 +1,11 @@
+---
+'@hyperdx/common-utils': minor
+---
+
+Fix legacy `filters` migration to merge with the existing `where` clause instead of replacing it, so filters are never dropped when both are present.
+
+- Add `mergeFilterStateIntoWhereClause` to AND existing `where` text with migrated filters (verbatim preservation, OR-wrapping of the residual, SQL-safe when needed).
+- Fix invalid/incomplete Lucene clauses (e.g. mid-edit `service:`) so `replaceLuceneFacetClauses` now emits the new clause instead of no-oping, and `replaceFilterClauses` reports them via `getWhereParseError`.
+- Support Lucene `NOT` negation (`NOT field:"v"`, `field:"v" AND NOT other:"w"`) in facet parsing so the sidebar shows an excluded (indeterminate) state instead of a checked one.
+- Add `getUnrepresentableWhereReason` so cross-field `OR` queries can be surfaced to the user rather than silently misrepresented as AND.
+- Add `emitLanguage` to `replaceFilterClauses` so facet clauses can be rewritten across query languages (Lucene ↔ SQL) while preserving non-facet text — used by the UI on language switch.

packages/app/src/DBSearchPage.tsx

@@ -116,7 +116,15 @@ import {
   useSavedSearch,
   useUpdateSavedSearch,
 } from '@/savedSearch';
-import { useSearchPageFilterState } from '@/searchFilters';
+import {
+  getUnrepresentableWhereReason,
+  getWhereParseError,
+  mergeFiltersIntoWhereClause,
+  replaceFiltersInWhereClause,
+  translateWhereClauseInQuery,
+  useSearchPageFilterState,
+  whereToFilters,
+} from '@/searchFilters';
 import { getEventBody, useSource, useSources } from '@/source';
 import { useAppTheme, useBrandDisplayName } from '@/theme/ThemeProvider';
 import {
@@ -1098,7 +1106,7 @@ export function DBSearchPage() {
     [sources, lastSelectedSourceId],
   );
 
-  const { control, setValue, reset, handleSubmit, formState } =
+  const { control, setValue, reset, getValues, handleSubmit, formState } =
     useForm<SearchConfigFromSchema>({
       values: {
         select: searchedConfig.select || '',
@@ -1263,18 +1271,15 @@ export function DBSearchPage() {
 
   const onSubmit = useCallback(() => {
     onSearch(displayedTimeInputValue);
-    handleSubmit(
-      ({ select, where, whereLanguage, source, filters, orderBy }) => {
-        setSearchedConfig({
-          select,
-          where,
-          whereLanguage,
-          source,
-          filters,
-          orderBy,
-        });
-      },
-    )();
+    handleSubmit(({ select, where, whereLanguage, source, orderBy }) => {
+      setSearchedConfig({
+        select,
+        where,
+        whereLanguage,
+        source,
+        orderBy,
+      });
+    })();
     setPatternColumn(draftPatternColumn || null);
     // clear query errors
     setQueryErrors({});
@@ -1289,13 +1294,6 @@ export function DBSearchPage() {
   ]);
 
   const debouncedSubmit = useDebouncedCallback(onSubmit, 1000);
-  const handleSetFilters = useCallback(
-    (filters: Filter[]) => {
-      setValue('filters', filters);
-      debouncedSubmit();
-    },
-    [debouncedSubmit, setValue],
-  );
 
   // Top-level column names for the active source, used to quote
   // filter keys that contain special characters.
@@ -1343,14 +1341,131 @@ export function DBSearchPage() {
   const { dateTimeColumns, onResolvedColumnsChange } =
     useResolvedDateTimeColumns(inputSourceColumns);
 
-  const filters = useWatch({ name: 'filters', control });
+  // The `where` clause is the canonical representation of the query. The
+  // sidebar's FilterState is derived from it (via `whereToFilters`) and sidebar
+  // toggles rewrite it in place (via `replaceFiltersInWhereClause`), so the
+  // filter sidebar and the query text can never drift apart.
+  const inputWhere = useWatch({ name: 'where', control });
+  const inputWhereLanguage = useWatch({ name: 'whereLanguage', control });
+
+  // The `where` text is the canonical representation, but while the user is
+  // mid-typing it can be momentarily unparseable (e.g. `service:`). Keep the
+  // last valid derived query so the sidebar doesn't wipe, and surface the
+  // parse error separately so the sidebar can explain what's happening.
+  const lastGoodSearchQueryRef = useRef<Filter[]>([]);
+  const searchQuery = useMemo(() => {
+    const language = inputWhereLanguage ?? 'lucene';
+    const parseError = getWhereParseError(inputWhere, language);
+    if (parseError) return lastGoodSearchQueryRef.current;
+    const query = whereToFilters(
+      inputWhere,
+      language,
+      knownColumns,
+      dateTimeColumns,
+    );
+    lastGoodSearchQueryRef.current = query;
+    return query;
+  }, [inputWhere, inputWhereLanguage, knownColumns, dateTimeColumns]);
+
+  const whereParseError = useMemo(
+    () => getWhereParseError(inputWhere, inputWhereLanguage ?? 'lucene'),
+    [inputWhere, inputWhereLanguage],
+  );
+
+  const whereUnrepresentableReason = useMemo(
+    () =>
+      getUnrepresentableWhereReason(inputWhere, inputWhereLanguage ?? 'lucene'),
+    [inputWhere, inputWhereLanguage],
+  );
+
+  // Language switches happen through SearchWhereInput; translate the live
+  // `where` text so facet clauses survive (the free-text/facet rewrite keeps
+  // non-facet content verbatim). Skip when there is nothing to translate.
+  const handleWhereLanguageChange = useCallback(
+    (language: 'sql' | 'lucene') => {
+      const previousLanguage = inputWhereLanguage ?? 'lucene';
+      if (language === previousLanguage) return;
+      const translatedWhere = translateWhereClauseInQuery(
+        inputWhere,
+        previousLanguage,
+        language,
+        knownColumns,
+        dateTimeColumns,
+      );
+      if (translatedWhere !== inputWhere) {
+        setValue('where', translatedWhere, { shouldDirty: true });
+      }
+    },
+    [inputWhere, inputWhereLanguage, knownColumns, dateTimeColumns, setValue],
+  );
+
+  // Sidebar filter mutations arrive as the canonical SQL `Filter[]` the hook
+  // emits; rewrite the facet clauses of the live `where` text to match and
+  // drop the separate `filters` param entirely.
+  //
+  // Read the current form values with `getValues()` (a non-reactive read) at
+  // call time instead of closing over the watched `where`, so this callback
+  // keeps a stable identity across keystrokes. A stable `onFilterChange` keeps
+  // the sidebar hook's eight mutators stable too, which is what lets
+  // `memo(DBSearchPageFiltersComponent)` skip re-rendering while typing.
+  const handleSetFilters = useCallback(
+    (filters: Filter[]) => {
+      const newWhere = replaceFiltersInWhereClause(
+        getValues('where') ?? '',
+        getValues('whereLanguage') ?? 'lucene',
+        filters,
+        knownColumns,
+        dateTimeColumns,
+      );
+      setValue('where', newWhere);
+      debouncedSubmit();
+    },
+    [debouncedSubmit, setValue, getValues, knownColumns, dateTimeColumns],
+  );
+
   const searchFilters = useSearchPageFilterState({
-    searchQuery: filters ?? undefined,
+    searchQuery,
     onFilterChange: handleSetFilters,
     dateTimeColumns,
     knownColumns,
   });
 
+  // One-time migration: legacy `filters` params (URL, saved search) move into
+  // the `where` clause, now the canonical representation. Gated on the source's
+  // columns being known so date columns and special-character keys emit
+  // correctly, then `filters` is cleared so the page stops persisting it.
+  useEffect(() => {
+    const { filters: legacyFilters, where, whereLanguage } = searchedConfig;
+    if (!legacyFilters?.length || !inputSourceColumns) return;
+    try {
+      const migratedWhere = mergeFiltersIntoWhereClause(
+        where ?? '',
+        (whereLanguage as 'sql' | 'lucene') ?? 'lucene',
+        legacyFilters,
+        knownColumns,
+        dateTimeColumns,
+      );
+      if (migratedWhere !== (where ?? '')) {
+        setSearchedConfig({ where: migratedWhere, filters: [] });
+      } else if (where) {
+        // Filters were already represented in the where text — just stop
+        // persisting the redundant param.
+        setSearchedConfig({ filters: [] });
+      }
+    } catch (e) {
+      console.error('Failed to migrate legacy filters into where clause', e);
+    }
+    // Runs when a legacy filters-bearing config is present; the emit depends on
+    // columns loading so re-check when they arrive.
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [
+    searchedConfig.filters,
+    searchedConfig.where,
+    searchedConfig.whereLanguage,
+    inputSourceColumns,
+    setSearchedConfig,
+  ]);
+
   useEffect(() => {
     // If the user changes the source dropdown, reset the select and orderby fields
     // to match the new source selected
@@ -1493,8 +1608,6 @@ export function DBSearchPage() {
       : null;
     return { hasQueryError, queryError };
   }, [_queryErrors]);
-  const inputWhere = useWatch({ name: 'where', control });
-  const inputWhereLanguage = useWatch({ name: 'whereLanguage', control });
   // query suggestion for 'where' if error
   const whereSuggestions = useSqlSuggestions({
     input: inputWhere,
@@ -1712,15 +1825,13 @@ export function DBSearchPage() {
       } else {
         qParams.append('select', searchedConfig.select || '');
         qParams.append('where', where || searchedConfig.where || '');
-        qParams.append('filters', JSON.stringify(searchedConfig.filters ?? []));
         qParams.append('source', searchedSource?.id || '');
       }
 
       return `/search?${qParams.toString()}`;
     },
     [
       interval,
-      searchedConfig.filters,
       searchedConfig.select,
       searchedConfig.where,
       searchedSource?.id,
@@ -2314,6 +2425,7 @@ export function DBSearchPage() {
             control={control}
             name="where"
             onSubmit={onSubmit}
+            onLanguageChange={handleWhereLanguageChange}
             sqlQueryHistoryType={QUERY_LOCAL_STORAGE.SEARCH_SQL}
             luceneQueryHistoryType={QUERY_LOCAL_STORAGE.SEARCH_LUCENE}
             enableHotkey
@@ -2431,6 +2543,8 @@ export function DBSearchPage() {
                     onColumnToggle={toggleColumn}
                     displayedColumns={displayedColumns}
                     onCollapse={() => setIsFilterSidebarCollapsed(true)}
+                    whereParseError={whereParseError}
+                    whereUnrepresentableReason={whereUnrepresentableReason}
                     {...searchFilters}
                   />
                 </ErrorBoundary>

packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx

@@ -118,6 +118,12 @@ jest.mock('@/searchFilters', () => ({
     setFilterValue: jest.fn(),
     clearAllFilters: jest.fn(),
   }),
+  whereToFilters: () => [],
+  replaceFiltersInWhereClause: (where: string) => where,
+  mergeFiltersIntoWhereClause: (where: string) => where,
+  translateWhereClauseInQuery: (where: string) => where,
+  getWhereParseError: () => null,
+  getUnrepresentableWhereReason: () => null,
 }));
 
 jest.mock('@/hooks/useChartConfig', () => ({

packages/app/src/__tests__/SessionSidePanel.test.tsx

@@ -40,7 +40,7 @@ jest.mock('nuqs', () => {
   const noop = () => {};
   return {
     ...actual,
-    // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix
+
     useQueryState: (key: string, parser?: { defaultValue?: unknown }) =>
       key === 'sessionPanelEvent'
         ? [mockNuqs.sessionPanelEvent, mockNuqs.setSessionPanelEvent]

packages/app/src/components/DBSearchPageFilters.tsx

@@ -12,6 +12,7 @@ import {
 import {
   Accordion,
   ActionIcon,
+  Alert,
   Box,
   Button,
   Center,
@@ -33,6 +34,7 @@ import {
 } from '@mantine/core';
 import { notifications } from '@mantine/notifications';
 import {
+  IconAlertCircle,
   IconArrowBarToLeft,
   IconChartBar,
   IconChartBarOff,
@@ -1072,6 +1074,8 @@ const DBSearchPageFiltersComponent = ({
   onColumnToggle,
   displayedColumns,
   onCollapse,
+  whereParseError,
+  whereUnrepresentableReason,
 }: {
   analysisMode: 'results' | 'delta' | 'pattern';
   setAnalysisMode: (mode: 'results' | 'delta' | 'pattern') => void;
@@ -1085,6 +1089,8 @@ const DBSearchPageFiltersComponent = ({
   onColumnToggle?: (column: string) => void;
   displayedColumns?: string[];
   onCollapse?: () => void;
+  whereParseError?: string | null;
+  whereUnrepresentableReason?: string | null;
 } & FilterStateHook) => {
   const setFilterValue = useCallback(
     (
@@ -1662,6 +1668,35 @@ const DBSearchPageFiltersComponent = ({
               )}
             </Group>
           </Flex>
+          {whereParseError && (
+            <Alert
+              variant="light"
+              color="orange"
+              radius="sm"
+              p="xs"
+              title="Query is incomplete"
+              icon={<IconAlertCircle size={16} />}
+            >
+              <Text size="xs" c="dimmed">
+                Finish the query text above, or click a value below to replace
+                the incomplete text.
+              </Text>
+            </Alert>
+          )}
+          {!whereParseError && whereUnrepresentableReason && (
+            <Alert
+              variant="light"
+              color="orange"
+              radius="sm"
+              p="xs"
+              title="Filters shown partially"
+              icon={<IconAlertCircle size={16} />}
+            >
+              <Text size="xs" c="dimmed">
+                {whereUnrepresentableReason}
+              </Text>
+            </Alert>
+          )}
           <Tabs
             value={analysisMode}
             onChange={value =>

packages/app/src/components/__tests__/DBRowSidePanel.missingSource.test.tsx

@@ -26,7 +26,7 @@ jest.mock('nuqs', () => {
   const actual = jest.requireActual('nuqs');
   return {
     ...actual,
-    // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix
+
     useQueryState: (key: string, parser?: { defaultValue?: unknown }) => {
       const hasValue = Object.prototype.hasOwnProperty.call(
         mockQueryStore,

packages/app/src/components/__tests__/DBRowSidePanel.spanLinks.test.tsx

@@ -24,7 +24,7 @@ jest.mock('nuqs', () => {
   const actual = jest.requireActual('nuqs');
   return {
     ...actual,
-    // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix
+
     useQueryState: (key: string, parser?: { defaultValue?: unknown }) => {
       const hasValue = Object.prototype.hasOwnProperty.call(
         mockQueryStore,
@@ -56,7 +56,7 @@ const LINK = {
 const mockUseRowData = jest.fn();
 jest.mock('../DBRowDataPanel', () => ({
   __esModule: true,
-  // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix
+
   useRowData: (args: unknown) => mockUseRowData(args),
   ROW_DATA_ALIASES: {
     DURATION_MS: '__hdx_duration',
@@ -82,14 +82,14 @@ const TRACE_SOURCE = {
 jest.mock('@/source', () => ({
   __esModule: true,
   getEventBody: () => undefined,
-  // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix
+
   useSource: ({ id }: { id: string | null }) =>
     id === 'trace-src' ? { data: TRACE_SOURCE } : { data: undefined },
 }));
 
 jest.mock('../DBSessionPanel', () => ({
   __esModule: true,
-  // eslint-disable-next-line @eslint-react/no-unnecessary-use-prefix
+
   useSessionId: () => ({ rumSessionId: undefined, rumServiceName: undefined }),
   DBSessionPanel: () => null,
 }));