Skip to content

Commit be03225

Browse files
committed
added envvar resolution, fixed UI for the data drains, added screenshots to docs
1 parent ea1c7b4 commit be03225

21 files changed

Lines changed: 510 additions & 168 deletions

File tree

apps/docs/content/docs/en/enterprise/data-drains.mdx

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,10 @@ Drains are independent of [Data Retention](/enterprise/data-retention) but desig
1515

1616
Go to **Settings → Enterprise → Data Drains** in your workspace, then click **New drain**.
1717

18+
![Data Drains settings page showing two configured drains — one exporting workflow logs to Amazon S3 daily, another exporting Copilot chats to an HTTPS webhook hourly](/static/enterprise/data-drains-list.png)
19+
20+
![New data drain dialog with fields for name, source, cadence, destination, and S3 credentials](/static/enterprise/data-drains-new.png)
21+
1822
Each drain has four pieces:
1923

2024
1. A **source** — the category of data to export
51.9 KB
Loading
45.2 KB
Loading

apps/sim/app/api/organizations/[id]/data-drains/[drainId]/route.ts

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,11 @@ import {
1212
} from '@/lib/api/contracts/data-drains'
1313
import { parseRequest, validationErrorResponse } from '@/lib/api/server'
1414
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
15-
import { authorizeDrainAccess, loadDrain } from '@/lib/data-drains/access'
15+
import {
16+
authorizeDrainAccess,
17+
loadDrain,
18+
validateDrainEnvironmentWorkspace,
19+
} from '@/lib/data-drains/access'
1620
import { getDestination } from '@/lib/data-drains/destinations/registry'
1721
import { encryptCredentials } from '@/lib/data-drains/encryption'
1822
import { serializeDrain } from '@/lib/data-drains/serializers'
@@ -75,10 +79,19 @@ export const PUT = withRouteHandler(async (request: NextRequest, context: RouteC
7579
return NextResponse.json({ error: 'source cannot be changed after creation' }, { status: 400 })
7680
}
7781

82+
const envWorkspaceError = await validateDrainEnvironmentWorkspace(
83+
organizationId,
84+
body.environmentWorkspaceId
85+
)
86+
if (envWorkspaceError) return envWorkspaceError
87+
7888
const updates: Partial<typeof dataDrains.$inferInsert> = { updatedAt: new Date() }
7989
if (body.name !== undefined) updates.name = body.name
8090
if (body.scheduleCadence !== undefined) updates.scheduleCadence = body.scheduleCadence
8191
if (body.enabled !== undefined) updates.enabled = body.enabled
92+
if (body.environmentWorkspaceId !== undefined) {
93+
updates.environmentWorkspaceId = body.environmentWorkspaceId
94+
}
8295

8396
if (body.destinationType !== undefined && body.destinationType !== drain.destinationType) {
8497
return NextResponse.json(
@@ -141,6 +154,7 @@ export const PUT = withRouteHandler(async (request: NextRequest, context: RouteC
141154
source: body.source,
142155
scheduleCadence: body.scheduleCadence,
143156
enabled: body.enabled,
157+
environmentWorkspaceId: body.environmentWorkspaceId,
144158
destinationConfigChanged: body.destinationConfig !== undefined,
145159
destinationCredentialsChanged: body.destinationCredentials !== undefined,
146160
},

apps/sim/app/api/organizations/[id]/data-drains/[drainId]/test/route.ts

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
88
import { authorizeDrainAccess, loadDrain } from '@/lib/data-drains/access'
99
import { getDestination } from '@/lib/data-drains/destinations/registry'
1010
import { decryptCredentials } from '@/lib/data-drains/encryption'
11+
import { resolveDataDrainDestinationEnvVars } from '@/lib/data-drains/resolve-config'
1112

1213
const logger = createLogger('DataDrainTestAPI')
1314

@@ -36,10 +37,16 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Route
3637
)
3738
}
3839

39-
const config = destination.configSchema.parse(drain.destinationConfig)
40-
const credentials = destination.credentialsSchema.parse(
41-
await decryptCredentials(drain.destinationCredentials)
40+
const unresolvedConfig = destination.configSchema.parse(drain.destinationConfig)
41+
const unresolvedCredentials = await decryptCredentials(drain.destinationCredentials)
42+
const resolved = await resolveDataDrainDestinationEnvVars(
43+
unresolvedConfig,
44+
unresolvedCredentials,
45+
drain.createdBy,
46+
drain.environmentWorkspaceId ?? undefined
4247
)
48+
const config = destination.configSchema.parse(resolved.config)
49+
const credentials = destination.credentialsSchema.parse(resolved.credentials)
4350

4451
const controller = new AbortController()
4552
const timeout = setTimeout(() => controller.abort(), TEST_TIMEOUT_MS)

apps/sim/app/api/organizations/[id]/data-drains/route.ts

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import { type NextRequest, NextResponse } from 'next/server'
99
import { createDataDrainContract, listDataDrainsContract } from '@/lib/api/contracts/data-drains'
1010
import { parseRequest, validationErrorResponse } from '@/lib/api/server'
1111
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
12-
import { authorizeDrainAccess } from '@/lib/data-drains/access'
12+
import { authorizeDrainAccess, validateDrainEnvironmentWorkspace } from '@/lib/data-drains/access'
1313
import { getDestination } from '@/lib/data-drains/destinations/registry'
1414
import { encryptCredentials } from '@/lib/data-drains/encryption'
1515
import { serializeDrain } from '@/lib/data-drains/serializers'
@@ -45,6 +45,12 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Route
4545

4646
const body = parsed.data.body
4747

48+
const envWorkspaceError = await validateDrainEnvironmentWorkspace(
49+
organizationId,
50+
body.environmentWorkspaceId
51+
)
52+
if (envWorkspaceError) return envWorkspaceError
53+
4854
if (!body.destinationCredentials) {
4955
return NextResponse.json(
5056
{ error: 'destinationCredentials is required when creating a drain' },
@@ -84,6 +90,7 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Route
8490
destinationType: body.destinationType,
8591
destinationConfig: configResult.data as Record<string, unknown>,
8692
destinationCredentials: encryptedCredentials,
93+
environmentWorkspaceId: body.environmentWorkspaceId,
8794
scheduleCadence: body.scheduleCadence,
8895
enabled: body.enabled ?? true,
8996
cursor: null,

0 commit comments

Comments
 (0)