perf(prefetch): read the data layer instead of calling our own API over the wire

waleedlatif1 · waleedlatif1 · commit 0fa05c823e57 · 2026-08-12T21:27:41.000-07:00
Four server-render prefetches went out over HTTP to our own routes. With
INTERNAL_API_BASE_URL unset in prod, getInternalApiBaseUrl() falls back to the
public base URL, so each was RSC -&gt; public HTTPS -&gt; load balancer -&gt; back into
the app, awaited inside the render with a second round of auth.

- /home fetched the workflow folder list that the workspace layout had already
  fetched, under the identical query key. Since getQueryClient() builds a new
  client per call on the server, the two never deduped: same data, twice a
  request, once directly and once over the wire. Dropped; the layout's entry
  already hydrates it.
- /home cached raw route JSON under workspaceFilesKeys.list, while
  files/prefetch.ts seeds that same key from listWorkspaceFilesWithShares. The
  contract declares the date fields z.coerce.date(), so consumers hold Dates —
  a file record's type depended on which page the viewer landed on. Now reads
  the same function files/prefetch.ts does.
- tables and knowledge folder reads now call listFoldersForWorkspace, matching
  the sidebar prefetch.

These reads carry no authorization of their own, so each surface proves the
viewer through getWorkspaceHostContextForViewer first and caches nothing when
it fails, leaving the client fetch to reach the route for the real 403. Both it
and getSession are cache()d and already resolved by the layout, so the proof
costs no extra queries.

Left on the wire, deliberately: the tables and knowledge lists, whose cached
shape is the serialized wire shape, and pinned items and members, which have no
exported data-layer function.
diff --git a/apps/sim/app/workspace/[workspaceId]/home/page.tsx b/apps/sim/app/workspace/[workspaceId]/home/page.tsx
@@ -24,10 +24,18 @@ export default async function HomePage({ params }: { params: Promise<{ workspace
   }
 
   const queryClient = getQueryClient()
-  const listsPrefetch = prefetchHomeLists(queryClient, workspaceId)
 
+  /**
+   * `getSession` is `cache`d and the layout has already resolved it for this
+   * request, so awaiting it before the prefetch costs nothing and gives the
+   * prefetch the viewer it needs to authorize its own read.
+   */
   const session = await getSession()
   const userId = session?.user?.id
+  const listsPrefetch = userId
+    ? prefetchHomeLists(queryClient, workspaceId, userId)
+    : Promise.resolve()
+
   const tableViewsEnabled = await resolveTableViewsEnabled(workspaceId, userId)
   await listsPrefetch
 
diff --git a/apps/sim/app/workspace/[workspaceId]/home/prefetch.ts b/apps/sim/app/workspace/[workspaceId]/home/prefetch.ts
@@ -1,50 +1,45 @@
 import type { QueryClient } from '@tanstack/react-query'
-import type { FolderApi } from '@/lib/api/contracts'
-import type { ListWorkspaceFilesResponse } from '@/lib/api/contracts/workspace-files'
-import { prefetchInternalJson } from '@/app/workspace/[workspaceId]/lib/prefetch-internal-fetch'
-import { FOLDER_LIST_STALE_TIME, folderKeys, mapFolder } from '@/hooks/queries/utils/folder-keys'
+import { listWorkspaceFilesWithShares } from '@/lib/workspace-files/queries'
+import { getWorkspaceHostContextForViewer } from '@/lib/workspaces/host-context'
 import {
   WORKSPACE_FILES_LIST_STALE_TIME,
   workspaceFilesKeys,
 } from '@/hooks/queries/workspace-files'
 
 /**
- * Prefetches the home page's secondary lists — folders and workspace files —
- * under the same query keys their client hooks (`useFolders`,
- * `useWorkspaceFiles`) use, so the home view paints populated on first render.
+ * Prefetches the workspace files the home view lists, under the same query key
+ * its client hook (`useWorkspaceFiles`) uses, so the view paints populated on
+ * first render.
  *
- * The workflow list (`workflowKeys.list(ws, 'active')`) is already hydrated by
- * the workspace sidebar prefetch and is intentionally not repeated here.
+ * Reads the data layer rather than the route, which drops a server-to-server
+ * request and its duplicate auth. It also fixes the shape this key was seeded
+ * with: `listWorkspaceFilesContract` declares the date fields as
+ * `z.coerce.date()`, so every consumer of `workspaceFilesKeys.list` holds
+ * `Date`s, and `files/prefetch.ts` already seeds them that way from this same
+ * function. Caching the raw route JSON here put ISO strings under that key
+ * instead, so a file record's type depended on which page the viewer landed on.
  *
- * Folders are fetched through the route and mapped with the same `mapFolder`
- * the hook applies, matching its cached shape (string dates → `Date`). Files
- * carry `Date` fields, so they go through the route and cache the serialized
- * wire shape — see {@link prefetchInternalJson}.
+ * The read carries no authorization of its own, so the viewer is proved first.
+ * `getWorkspaceHostContextForViewer` is `cache`d and the layout has already
+ * resolved it for this request, so this costs no additional queries; a viewer
+ * without access caches nothing and the client fetch reaches the route for the
+ * real 403.
+ *
+ * Folders (`folderKeys.list(ws, 'active', 'workflow')`) and the workflow list
+ * are both already hydrated by the workspace sidebar prefetch and are
+ * intentionally not repeated here.
  */
 export async function prefetchHomeLists(
   queryClient: QueryClient,
-  workspaceId: string
+  workspaceId: string,
+  userId: string
 ): Promise<void> {
-  await Promise.all([
-    queryClient.prefetchQuery({
-      queryKey: folderKeys.list(workspaceId, 'active', 'workflow'),
-      queryFn: async () => {
-        const { folders } = await prefetchInternalJson<{ folders?: FolderApi[] }>(
-          `/api/folders?workspaceId=${workspaceId}&scope=active&resourceType=workflow`
-        )
-        return (folders ?? []).map(mapFolder)
-      },
-      staleTime: FOLDER_LIST_STALE_TIME,
-    }),
-    queryClient.prefetchQuery({
-      queryKey: workspaceFilesKeys.list(workspaceId, 'active'),
-      queryFn: async () => {
-        const data = await prefetchInternalJson<ListWorkspaceFilesResponse>(
-          `/api/workspaces/${workspaceId}/files?scope=active`
-        )
-        return data.success ? data.files : []
-      },
-      staleTime: WORKSPACE_FILES_LIST_STALE_TIME,
-    }),
-  ])
+  const hostContext = await getWorkspaceHostContextForViewer(workspaceId, userId)
+  if (!hostContext) return
+
+  await queryClient.prefetchQuery({
+    queryKey: workspaceFilesKeys.list(workspaceId, 'active'),
+    queryFn: () => listWorkspaceFilesWithShares(workspaceId, 'active'),
+    staleTime: WORKSPACE_FILES_LIST_STALE_TIME,
+  })
 }
diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/page.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/page.tsx
@@ -1,6 +1,7 @@
 import { Suspense } from 'react'
 import { dehydrate, HydrationBoundary } from '@tanstack/react-query'
 import type { Metadata } from 'next'
+import { getSession } from '@/lib/auth'
 import { getQueryClient } from '@/app/_shell/providers/get-query-client'
 import { FOLDERED_RESOURCE_HEADERS } from '@/app/workspace/[workspaceId]/components/folders/foldered-resources'
 import { Knowledge } from '@/app/workspace/[workspaceId]/knowledge/knowledge'
@@ -27,7 +28,13 @@ export default async function KnowledgePage({
   const { workspaceId } = await params
 
   const queryClient = getQueryClient()
-  await prefetchKnowledgeBases(queryClient, workspaceId)
+  /**
+   * `getSession` is `cache`d and the layout has already resolved it for this
+   * request, so this costs nothing and gives the prefetch the viewer its
+   * data-layer reads need to authorize against.
+   */
+  const session = await getSession()
+  await prefetchKnowledgeBases(queryClient, workspaceId, session?.user?.id ?? '')
 
   return (
     <HydrationBoundary state={dehydrate(queryClient)}>
diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/prefetch.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/prefetch.ts
@@ -1,6 +1,7 @@
 import type { QueryClient } from '@tanstack/react-query'
-import type { FolderApi } from '@/lib/api/contracts/folders'
 import type { KnowledgeBaseData } from '@/lib/api/contracts/knowledge'
+import { listFoldersForWorkspace } from '@/lib/folders/queries'
+import { getWorkspaceHostContextForViewer } from '@/lib/workspaces/host-context'
 import { prefetchInternalJson } from '@/app/workspace/[workspaceId]/lib/prefetch-internal-fetch'
 import { prefetchResourceListChrome } from '@/app/workspace/[workspaceId]/lib/prefetch-resource-list-chrome'
 import { FOLDER_LIST_STALE_TIME, folderKeys, mapFolder } from '@/hooks/queries/utils/folder-keys'
@@ -16,14 +17,23 @@ import { KNOWLEDGE_BASE_LIST_STALE_TIME, knowledgeKeys } from '@/hooks/queries/u
  * beside, so prefetching one without the other still flashes an ungrouped list — and a
  * `?folderId=` deep link renders an empty breadcrumb until the folders arrive.
  *
- * The list carries `Date` fields, so it goes through the `/api/knowledge` route and caches the
- * serialized wire shape — see {@link prefetchInternalJson}. Folders are mapped with the same
- * `mapFolder` the hook applies, so the hydrated entry matches a client fetch exactly.
+ * Folders read the data layer and are mapped with the same `mapFolder` the hook applies,
+ * matching the workspace sidebar prefetch. That read carries no authorization of its own, so
+ * the viewer is proved first; `getWorkspaceHostContextForViewer` is `cache`d and the layout has
+ * already resolved it for this request, so it costs no additional queries.
+ *
+ * The bases list still goes through the `/api/knowledge` route — see
+ * {@link prefetchInternalJson}. It is served by an application use case that authorizes against
+ * a `Principal`, so converting it means constructing that principal here rather than reading a
+ * manager directly.
  */
 export async function prefetchKnowledgeBases(
   queryClient: QueryClient,
-  workspaceId: string
+  workspaceId: string,
+  userId: string
 ): Promise<void> {
+  const hostContext = await getWorkspaceHostContextForViewer(workspaceId, userId)
+
   await Promise.all([
     queryClient.prefetchQuery({
       queryKey: knowledgeKeys.list(workspaceId, 'active'),
@@ -35,16 +45,18 @@ export async function prefetchKnowledgeBases(
       },
       staleTime: KNOWLEDGE_BASE_LIST_STALE_TIME,
     }),
-    queryClient.prefetchQuery({
-      queryKey: folderKeys.list(workspaceId, 'active', 'knowledge_base'),
-      queryFn: async () => {
-        const { folders } = await prefetchInternalJson<{ folders?: FolderApi[] }>(
-          `/api/folders?workspaceId=${workspaceId}&scope=active&resourceType=knowledge_base`
-        )
-        return (folders ?? []).map(mapFolder)
-      },
-      staleTime: FOLDER_LIST_STALE_TIME,
-    }),
+    ...(hostContext
+      ? [
+          queryClient.prefetchQuery({
+            queryKey: folderKeys.list(workspaceId, 'active', 'knowledge_base'),
+            queryFn: async () => {
+              const rows = await listFoldersForWorkspace(workspaceId, 'active', 'knowledge_base')
+              return rows.map(mapFolder)
+            },
+            staleTime: FOLDER_LIST_STALE_TIME,
+          }),
+        ]
+      : []),
     prefetchResourceListChrome(queryClient, workspaceId, 'knowledge_base'),
   ])
 }
diff --git a/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts b/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts
@@ -6,11 +6,13 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'
 
 const {
   mockGetWorkspaceHostContextForViewer,
+  mockListFoldersForWorkspace,
   mockListWorkspaceFileFolders,
   mockListWorkspaceFilesWithShares,
   mockPrefetchInternalJson,
 } = vi.hoisted(() => ({
   mockGetWorkspaceHostContextForViewer: vi.fn(),
+  mockListFoldersForWorkspace: vi.fn(),
   mockListWorkspaceFileFolders: vi.fn(),
   mockListWorkspaceFilesWithShares: vi.fn(),
   mockPrefetchInternalJson: vi.fn(),
@@ -19,6 +21,9 @@ const {
 vi.mock('@/lib/workspaces/host-context', () => ({
   getWorkspaceHostContextForViewer: mockGetWorkspaceHostContextForViewer,
 }))
+vi.mock('@/lib/folders/queries', () => ({
+  listFoldersForWorkspace: mockListFoldersForWorkspace,
+}))
 vi.mock('@/lib/workspace-files/queries', () => ({
   listWorkspaceFilesWithShares: mockListWorkspaceFilesWithShares,
 }))
@@ -57,17 +62,79 @@ describe('workspace list prefetches', () => {
   beforeEach(() => {
     vi.clearAllMocks()
     mockGetWorkspaceHostContextForViewer.mockResolvedValue({ viewer: { permission: 'admin' } })
+    mockListFoldersForWorkspace.mockResolvedValue([])
     mockListWorkspaceFilesWithShares.mockResolvedValue([])
     mockListWorkspaceFileFolders.mockResolvedValue([])
   })
 
+  describe.each([
+    {
+      name: 'prefetchKnowledgeBases',
+      run: (client: QueryClient) => prefetchKnowledgeBases(client, WORKSPACE_ID, USER_ID),
+      resourceType: 'knowledge_base' as const,
+    },
+    {
+      name: 'prefetchTables',
+      run: (client: QueryClient) => prefetchTables(client, WORKSPACE_ID, USER_ID),
+      resourceType: 'table' as const,
+    },
+  ])('$name folder reads', ({ run, resourceType }) => {
+    it('reads folders from the data layer rather than over the wire', async () => {
+      const folderRow = {
+        id: 'fld-1',
+        name: 'Folder',
+        userId: 'u-1',
+        workspaceId: WORKSPACE_ID,
+        parentId: null,
+        resourceType,
+        locked: false,
+        sortOrder: 0,
+        createdAt: '2026-01-01T00:00:00.000Z',
+        updatedAt: '2026-01-02T00:00:00.000Z',
+        deletedAt: null,
+      }
+      mockPrefetchInternalJson.mockResolvedValue({ data: { tables: [] } })
+      mockListFoldersForWorkspace.mockResolvedValue([folderRow])
+      const client = makeClient()
+
+      await run(client)
+
+      expect(mockListFoldersForWorkspace).toHaveBeenCalledWith(WORKSPACE_ID, 'active', resourceType)
+      expect(mockPrefetchInternalJson).not.toHaveBeenCalledWith(
+        expect.stringContaining('/api/folders')
+      )
+      const cached = client.getQueryData(
+        folderKeys.list(WORKSPACE_ID, 'active', resourceType)
+      ) as Array<{
+        resourceType: string
+        createdAt: Date
+      }>
+      expect(cached).toHaveLength(1)
+      expect(cached[0].resourceType).toBe(resourceType)
+      expect(cached[0].createdAt).toBeInstanceOf(Date)
+    })
+
+    it('skips the folder read when the viewer cannot be proved', async () => {
+      mockPrefetchInternalJson.mockResolvedValue({ data: { tables: [] } })
+      mockGetWorkspaceHostContextForViewer.mockResolvedValue(null)
+      const client = makeClient()
+
+      await run(client)
+
+      expect(mockListFoldersForWorkspace).not.toHaveBeenCalled()
+      expect(
+        client.getQueryData(folderKeys.list(WORKSPACE_ID, 'active', resourceType))
+      ).toBeUndefined()
+    })
+  })
+
   describe('prefetchKnowledgeBases', () => {
     it('primes the exact key useKnowledgeBasesQuery reads and unwraps data', async () => {
       const bases = [{ id: 'kb-1' }]
       mockPrefetchInternalJson.mockResolvedValue({ data: bases })
       const client = makeClient()
 
-      await prefetchKnowledgeBases(client, WORKSPACE_ID)
+      await prefetchKnowledgeBases(client, WORKSPACE_ID, USER_ID)
 
       expect(mockPrefetchInternalJson).toHaveBeenCalledWith(
         `/api/knowledge?workspaceId=${WORKSPACE_ID}&scope=active`
@@ -82,7 +149,7 @@ describe('workspace list prefetches', () => {
       mockPrefetchInternalJson.mockResolvedValue({ data: { tables } })
       const client = makeClient()
 
-      await prefetchTables(client, WORKSPACE_ID)
+      await prefetchTables(client, WORKSPACE_ID, USER_ID)
 
       expect(mockPrefetchInternalJson).toHaveBeenCalledWith(
         `/api/table?workspaceId=${WORKSPACE_ID}&scope=active`
@@ -138,12 +205,12 @@ describe('workspace list prefetches', () => {
       },
       {
         name: 'tables',
-        run: (client: QueryClient) => prefetchTables(client, WORKSPACE_ID),
+        run: (client: QueryClient) => prefetchTables(client, WORKSPACE_ID, USER_ID),
         resourceType: 'table' as const,
       },
       {
         name: 'knowledge',
-        run: (client: QueryClient) => prefetchKnowledgeBases(client, WORKSPACE_ID),
+        run: (client: QueryClient) => prefetchKnowledgeBases(client, WORKSPACE_ID, USER_ID),
         resourceType: 'knowledge_base' as const,
       },
     ]
@@ -203,20 +270,20 @@ describe('workspace list prefetches', () => {
       )
       const client = makeClient()
 
-      await prefetchHomeLists(client, WORKSPACE_ID)
+      mockListWorkspaceFilesWithShares.mockResolvedValue(files)
 
-      expect(mockPrefetchInternalJson).toHaveBeenCalledWith(
-        `/api/folders?workspaceId=${WORKSPACE_ID}&scope=active&resourceType=workflow`
-      )
-      const cachedFolders = client.getQueryData(folderKeys.list(WORKSPACE_ID, 'active')) as Array<{
-        id: string
-        resourceType: string
-        createdAt: Date
-      }>
-      expect(cachedFolders).toHaveLength(1)
-      expect(cachedFolders[0].resourceType).toBe('workflow')
-      // The wire shape carries ISO strings; the client shape carries Dates.
-      expect(cachedFolders[0].createdAt).toBeInstanceOf(Date)
+      await prefetchHomeLists(client, WORKSPACE_ID, USER_ID)
+
+      /**
+       * Folders are hydrated by the workspace sidebar prefetch under this same
+       * key, so repeating them here would be a second read of data the layout
+       * already has.
+       */
+      expect(mockPrefetchInternalJson).not.toHaveBeenCalled()
+      expect(mockListFoldersForWorkspace).not.toHaveBeenCalled()
+      expect(client.getQueryData(folderKeys.list(WORKSPACE_ID, 'active'))).toBeUndefined()
+
+      expect(mockListWorkspaceFilesWithShares).toHaveBeenCalledWith(WORKSPACE_ID, 'active')
       expect(client.getQueryData(workspaceFilesKeys.list(WORKSPACE_ID, 'active'))).toEqual(files)
     })
   })
@@ -225,17 +292,17 @@ describe('workspace list prefetches', () => {
     it.each([
       [
         'prefetchKnowledgeBases',
-        (client: QueryClient) => prefetchKnowledgeBases(client, WORKSPACE_ID),
+        (client: QueryClient) => prefetchKnowledgeBases(client, WORKSPACE_ID, USER_ID),
         knowledgeKeys.list(WORKSPACE_ID, 'active'),
       ],
       [
         'prefetchTables',
-        (client: QueryClient) => prefetchTables(client, WORKSPACE_ID),
+        (client: QueryClient) => prefetchTables(client, WORKSPACE_ID, USER_ID),
         tableKeys.list(WORKSPACE_ID, 'active'),
       ],
       [
         'prefetchHomeLists',
-        (client: QueryClient) => prefetchHomeLists(client, WORKSPACE_ID),
+        (client: QueryClient) => prefetchHomeLists(client, WORKSPACE_ID, USER_ID),
         folderKeys.list(WORKSPACE_ID, 'active'),
       ],
       [
diff --git a/apps/sim/app/workspace/[workspaceId]/tables/page.tsx b/apps/sim/app/workspace/[workspaceId]/tables/page.tsx
@@ -1,6 +1,7 @@
 import { Suspense } from 'react'
 import { dehydrate, HydrationBoundary } from '@tanstack/react-query'
 import type { Metadata } from 'next'
+import { getSession } from '@/lib/auth'
 import { getQueryClient } from '@/app/_shell/providers/get-query-client'
 import TablesLoading from '@/app/workspace/[workspaceId]/tables/loading'
 import { prefetchTables } from '@/app/workspace/[workspaceId]/tables/prefetch'
@@ -20,7 +21,13 @@ export default async function TablesPage({ params }: { params: Promise<{ workspa
   const { workspaceId } = await params
 
   const queryClient = getQueryClient()
-  await prefetchTables(queryClient, workspaceId)
+  /**
+   * `getSession` is `cache`d and the layout has already resolved it for this
+   * request, so this costs nothing and gives the prefetch the viewer its
+   * data-layer reads need to authorize against.
+   */
+  const session = await getSession()
+  await prefetchTables(queryClient, workspaceId, session?.user?.id ?? '')
 
   return (
     <HydrationBoundary state={dehydrate(queryClient)}>
diff --git a/apps/sim/app/workspace/[workspaceId]/tables/prefetch.ts b/apps/sim/app/workspace/[workspaceId]/tables/prefetch.ts
