fix(v2): bind an AND-conjoined filter array as a set, not a sequence

The knowledge documents list fingerprinted tagFilters through canonicalJson,
which sorts object keys but preserves array order. Each filter compiles to a
condition in and(...whereConditions), and AND is commutative, so the same
clauses written in a different order select the same documents — and got a
different fingerprint, refusing a cursor for a page that was genuinely the
next one.

Adds unorderedJsonScopePart beside parseUnorderedList: members are
canonicalized, de-duplicated, and sorted, so `A AND A` binds like `A` and
clause order stops mattering. A non-array or unparseable value still binds
by its raw spelling, since that request fails validation anyway.

Replaces the route-local canonicalTagFilters, and corrects the claim on
canonicalJson that array order only ever costs a restart — for a set-valued
filter it costs a spurious 400.

Reported by Greptile.

Author: waleedlatif1

apps/sim/app/api/v2/knowledge/[id]/documents/route.ts

@@ -4,7 +4,7 @@ import {
   v2ListKnowledgeDocumentsContract,
   v2UploadKnowledgeDocumentContract,
 } from '@/lib/api/contracts/v2/knowledge'
-import { canonicalJson, cursorScopeKey } from '@/lib/api/cursor-binding'
+import { cursorScopeKey, unorderedJsonScopePart } from '@/lib/api/cursor-binding'
 import {
   defineV2BodyLifecycleRoute,
   defineV2JsonRoute,
@@ -40,21 +40,6 @@ export const revalidate = 0
 
 const MAX_FILE_SIZE = MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE
 
-/**
- * Canonical form of `tagFilters` so two equivalent filters differing only in
- * key order fingerprint the same. {@link canonicalJson} sorts object keys. An
- * unparseable value binds by its raw spelling — the request carrying it is
- * about to fail validation anyway.
- */
-function canonicalTagFilters(raw: string | undefined): string | undefined {
-  if (raw === undefined) return undefined
-  try {
-    return canonicalJson(JSON.parse(raw))
-  } catch {
-    return raw
-  }
-}
-
 /** Every param that changes which documents, in which order, this list returns. */
 function documentCursorFilters(
   knowledgeBaseId: string,
@@ -65,7 +50,7 @@ function documentCursorFilters(
     workspaceId: query.workspaceId,
     enabledFilter: query.enabledFilter,
     search: query.search,
-    tagFilters: canonicalTagFilters(query.tagFilters),
+    tagFilters: unorderedJsonScopePart(query.tagFilters),
   })
 }
 

apps/sim/lib/api/cursor-binding.test.ts

@@ -2,7 +2,12 @@
  * @vitest-environment node
  */
 import { describe, expect, it } from 'vitest'
-import { cursorScopeKey, parseUnorderedList, unorderedScopePart } from '@/lib/api/cursor-binding'
+import {
+  cursorScopeKey,
+  parseUnorderedList,
+  unorderedJsonScopePart,
+  unorderedScopePart,
+} from '@/lib/api/cursor-binding'
 import {
   cursorSortKey,
   decodeOffsetCursor,
@@ -219,6 +224,29 @@ describe('unordered filter scope parts', () => {
     expect(unorderedScopePart('A, B')).toBe(parseUnorderedList('A, B')?.join(','))
   })
 
+  /**
+   * Tag filters compile to `and(...)`, so reordering the clauses selects the
+   * same documents. Binding to the order a caller happened to write them in
+   * refused a cursor for a page that was genuinely the next one.
+   */
+  it('treats an AND-conjoined filter array as a set', () => {
+    const ab = '[{"name":"a","value":"1"},{"name":"b","value":"2"}]'
+    const ba = '[{"name":"b","value":"2"},{"name":"a","value":"1"}]'
+
+    expect(unorderedJsonScopePart(ab)).toBe(unorderedJsonScopePart(ba))
+    expect(unorderedJsonScopePart('[{"name":"a"},{"name":"a"}]')).toBe(
+      unorderedJsonScopePart('[{"name":"a"}]')
+    )
+    expect(unorderedJsonScopePart(ab)).not.toBe(
+      unorderedJsonScopePart('[{"name":"a","value":"1"}]')
+    )
+  })
+
+  it('binds an unparseable filter by its raw spelling', () => {
+    expect(unorderedJsonScopePart('{not json')).toBe('{not json')
+    expect(unorderedJsonScopePart(undefined)).toBeUndefined()
+  })
+
   it('still separates genuinely different sets', () => {
     expect(cursorScopeKey({ workflowIds: unorderedScopePart('A,B') })).not.toBe(
       cursorScopeKey({ workflowIds: unorderedScopePart('A,C') })

apps/sim/lib/api/cursor-binding.ts

@@ -94,13 +94,38 @@ export function parseUnorderedList(raw: string | undefined): string[] | undefine
   ].sort()
 }
 
+/**
+ * Canonical form of a JSON-encoded filter array whose members are AND-conjoined.
+ *
+ * {@link canonicalJson} preserves array order, which is right for a sequence but
+ * wrong for a set: clauses that compile to `and(...)` select the same rows in any
+ * order, so binding to the order a caller happened to write them refuses a cursor
+ * for a page that is genuinely the next one. Members are canonicalized, then
+ * de-duplicated and sorted — `A AND A` selects what `A` does.
+ *
+ * A non-array or unparseable value binds by its raw spelling: the request
+ * carrying it is about to fail validation anyway.
+ */
+export function unorderedJsonScopePart(raw: string | undefined): string | undefined {
+  if (raw === undefined) return undefined
+  try {
+    const parsed: unknown = JSON.parse(raw)
+    if (!Array.isArray(parsed)) return canonicalJson(parsed)
+    return `[${[...new Set(parsed.map(canonicalJson))].sort().join(',')}]`
+  } catch {
+    return raw
+  }
+}
+
 /**
  * Deterministic JSON: object keys sorted so two structurally equal values
  * serialize identically regardless of the key order they arrived in, and
  * `undefined` members dropped so an omitted param and an absent one agree.
  *
- * Array order is preserved — reordering an `in` list is treated as a different
- * filter, which only ever costs a restart.
+ * Array order is preserved, because an array is a sequence in the general case.
+ * A filter whose array is really a set must canonicalize it first — see
+ * {@link parseUnorderedList} and {@link unorderedJsonScopePart} — or equivalent
+ * queries fingerprint differently and a valid cursor is refused.
  */
 export function canonicalJson(value: unknown): string {
   if (value instanceof Date) return JSON.stringify(value.toISOString())