Skip to content

Commit 879dab9

Browse files
waleedlatif1waleedlatif1
authored
feat(table): make plan table limits configurable via env vars (#4406)
* feat(table): make plan table limits configurable via env vars * fix(table): coerce env table limits to number for skipValidation env * improvement(env): extract envNumber helper for numeric env coercion * improvement(knowledge): use envNumber helper for KB_CONFIG_* env reads * fix(testing): add envNumber to env mock factory * fix(env): allow zero in envNumber for max-throughput configs * fix(env): add min option to envNumber for strict-positive configs
1 parent 20ee07c commit 879dab9

12 files changed

Lines changed: 141 additions & 43 deletions

File tree

‎apps/sim/background/knowledge-processing.ts‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { createLogger } from '@sim/logger'
22
import { task } from '@trigger.dev/sdk'
3-
import { env } from '@/lib/core/config/env'
3+
import { env, envNumber } from '@/lib/core/config/env'
44
import { processDocumentAsync } from '@/lib/knowledge/documents/service'
55

66
const logger = createLogger('TriggerKnowledgeProcessing')
@@ -23,16 +23,16 @@ export type DocumentProcessingPayload = {
2323

2424
export const processDocument = task({
2525
id: 'knowledge-process-document',
26-
maxDuration: env.KB_CONFIG_MAX_DURATION || 600,
26+
maxDuration: envNumber(env.KB_CONFIG_MAX_DURATION, 600),
2727
machine: 'large-1x', // 2 vCPU, 2GB RAM - needed for large PDF processing
2828
retry: {
29-
maxAttempts: env.KB_CONFIG_MAX_ATTEMPTS || 3,
30-
factor: env.KB_CONFIG_RETRY_FACTOR || 2,
31-
minTimeoutInMs: env.KB_CONFIG_MIN_TIMEOUT || 1000,
32-
maxTimeoutInMs: env.KB_CONFIG_MAX_TIMEOUT || 10000,
29+
maxAttempts: envNumber(env.KB_CONFIG_MAX_ATTEMPTS, 3),
30+
factor: envNumber(env.KB_CONFIG_RETRY_FACTOR, 2),
31+
minTimeoutInMs: envNumber(env.KB_CONFIG_MIN_TIMEOUT, 1000),
32+
maxTimeoutInMs: envNumber(env.KB_CONFIG_MAX_TIMEOUT, 10000),
3333
},
3434
queue: {
35-
concurrencyLimit: env.KB_CONFIG_CONCURRENCY_LIMIT || 20,
35+
concurrencyLimit: envNumber(env.KB_CONFIG_CONCURRENCY_LIMIT, 20),
3636
name: 'document-processing-queue',
3737
},
3838
run: async (payload: DocumentProcessingPayload) => {

‎apps/sim/lib/billing/subscriptions/utils.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ import {
1414
isTeam,
1515
} from '@/lib/billing/plan-helpers'
1616
import { parseEnterpriseSubscriptionMetadata } from '@/lib/billing/types'
17-
import { env } from '@/lib/core/config/env'
17+
import { env, envNumber } from '@/lib/core/config/env'
1818

1919
export const ENTITLED_SUBSCRIPTION_STATUSES = ['active', 'past_due'] as const
2020

@@ -52,28 +52,28 @@ export function hasUsableSubscriptionAccess(
5252
* Get the free tier limit from env or fallback to default
5353
*/
5454
export function getFreeTierLimit(): number {
55-
return env.FREE_TIER_COST_LIMIT || DEFAULT_FREE_CREDITS
55+
return envNumber(env.FREE_TIER_COST_LIMIT, DEFAULT_FREE_CREDITS)
5656
}
5757

5858
/**
5959
* Get the pro tier limit from env or fallback to default
6060
*/
6161
export function getProTierLimit(): number {
62-
return env.PRO_TIER_COST_LIMIT || DEFAULT_PRO_TIER_COST_LIMIT
62+
return envNumber(env.PRO_TIER_COST_LIMIT, DEFAULT_PRO_TIER_COST_LIMIT)
6363
}
6464

6565
/**
6666
* Get the team tier limit per seat from env or fallback to default
6767
*/
6868
export function getTeamTierLimitPerSeat(): number {
69-
return env.TEAM_TIER_COST_LIMIT || DEFAULT_TEAM_TIER_COST_LIMIT
69+
return envNumber(env.TEAM_TIER_COST_LIMIT, DEFAULT_TEAM_TIER_COST_LIMIT)
7070
}
7171

7272
/**
7373
* Get the enterprise tier limit per seat from env or fallback to default
7474
*/
7575
export function getEnterpriseTierLimitPerSeat(): number {
76-
return env.ENTERPRISE_TIER_COST_LIMIT || DEFAULT_ENTERPRISE_TIER_COST_LIMIT
76+
return envNumber(env.ENTERPRISE_TIER_COST_LIMIT, DEFAULT_ENTERPRISE_TIER_COST_LIMIT)
7777
}
7878

7979
export function checkEnterprisePlan(subscription: any): boolean {

‎apps/sim/lib/billing/threshold-billing.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,12 @@ import {
1616
} from '@/lib/billing/subscriptions/utils'
1717
import { toDecimal, toNumber } from '@/lib/billing/utils/decimal'
1818
import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-handlers'
19-
import { env } from '@/lib/core/config/env'
19+
import { env, envNumber } from '@/lib/core/config/env'
2020
import { enqueueOutboxEvent } from '@/lib/core/outbox/service'
2121

2222
const logger = createLogger('ThresholdBilling')
2323

24-
const OVERAGE_THRESHOLD = env.OVERAGE_THRESHOLD_DOLLARS || DEFAULT_OVERAGE_THRESHOLD
24+
const OVERAGE_THRESHOLD = envNumber(env.OVERAGE_THRESHOLD_DOLLARS, DEFAULT_OVERAGE_THRESHOLD)
2525

2626
export async function checkAndBillOverageThreshold(userId: string): Promise<void> {
2727
try {

‎apps/sim/lib/copilot/request/session/buffer.ts‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { createLogger } from '@sim/logger'
22
import { toError } from '@sim/utils/errors'
33
import { sleep } from '@sim/utils/helpers'
4-
import { env } from '@/lib/core/config/env'
4+
import { env, envNumber } from '@/lib/core/config/env'
55
import { getRedisClient } from '@/lib/core/config/redis'
66
import {
77
type PersistedStreamEventEnvelope,
@@ -40,19 +40,11 @@ export type StreamConfig = {
4040

4141
export function getStreamConfig(): StreamConfig {
4242
return {
43-
ttlSeconds: parsePositiveNumber(env.COPILOT_STREAM_TTL_SECONDS, DEFAULT_TTL_SECONDS),
44-
eventLimit: parsePositiveNumber(env.COPILOT_STREAM_EVENT_LIMIT, DEFAULT_EVENT_LIMIT),
43+
ttlSeconds: envNumber(env.COPILOT_STREAM_TTL_SECONDS, DEFAULT_TTL_SECONDS, { min: 1 }),
44+
eventLimit: envNumber(env.COPILOT_STREAM_EVENT_LIMIT, DEFAULT_EVENT_LIMIT, { min: 1 }),
4545
}
4646
}
4747

48-
function parsePositiveNumber(value: number | string | undefined, fallback: number) {
49-
if (typeof value === 'number' && Number.isFinite(value) && value > 0) {
50-
return value
51-
}
52-
const parsed = Number(value)
53-
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback
54-
}
55-
5648
async function withRedisRetry<T>(
5749
metadata: RedisOperationMetadata,
5850
operation: (redis: NonNullable<ReturnType<typeof getRedisClient>>) => Promise<T>

‎apps/sim/lib/core/config/env.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,16 @@ export const env = createEnv({
6060
ENTERPRISE_STORAGE_LIMIT_GB: z.number().optional().default(500), // Default storage limit in GB for enterprise tier (can be overridden per org)
6161
BILLING_ENABLED: z.boolean().optional(), // Enable billing enforcement and usage tracking
6262

63+
// Table feature limits (per plan). Apply when billing is disabled (free tier defaults) or for billed plans.
64+
FREE_TABLES_LIMIT: z.number().optional(), // Max user tables per workspace on free tier (default: 3)
65+
FREE_TABLE_ROWS_LIMIT: z.number().optional(), // Max rows per table on free tier (default: 1000)
66+
PRO_TABLES_LIMIT: z.number().optional(), // Max user tables per workspace on pro tier (default: 25)
67+
PRO_TABLE_ROWS_LIMIT: z.number().optional(), // Max rows per table on pro tier (default: 5000)
68+
TEAM_TABLES_LIMIT: z.number().optional(), // Max user tables per workspace on team tier (default: 100)
69+
TEAM_TABLE_ROWS_LIMIT: z.number().optional(), // Max rows per table on team tier (default: 10000)
70+
ENTERPRISE_TABLES_LIMIT: z.number().optional(), // Max user tables per workspace on enterprise tier (default: 10000)
71+
ENTERPRISE_TABLE_ROWS_LIMIT: z.number().optional(), // Max rows per table on enterprise tier (default: 1000000)
72+
6373
// Credit-tier Stripe prices (monthly)
6474
STRIPE_PRICE_TIER_25_MO: z.string().min(1).optional(), // Pro: $25/mo (6,000 credits)
6575
STRIPE_PRICE_TIER_100_MO: z.string().min(1).optional(), // Max: $100/mo (25,000 credits)
@@ -504,3 +514,27 @@ export const isFalsy = (value: string | boolean | number | undefined) =>
504514
typeof value === 'string' ? value.toLowerCase() === 'false' || value === '0' : value === false
505515

506516
export { getEnv }
517+
518+
/**
519+
* Coerce an env-derived value to a finite number ≥ `min`, falling back to the
520+
* provided default when the value is unset, empty, non-finite, or below `min`.
521+
* `min` defaults to `0` so configs like `KB_CONFIG_DELAY_BETWEEN_BATCHES=0`
522+
* (meaning "no delay / max throughput") are honored. Pass `min: 1` for configs
523+
* where zero is invalid (e.g. Redis TTLs, capacity limits).
524+
*
525+
* `createEnv` is configured with `skipValidation: true`, so values declared as
526+
* `z.number()` arrive as raw strings when sourced from `process.env` or Helm.
527+
* Use this helper anywhere a numeric env override is consumed to normalize the
528+
* type at the boundary instead of relying on JS implicit coercion.
529+
*/
530+
export function envNumber(
531+
value: number | string | undefined | null,
532+
fallback: number,
533+
options: { min?: number } = {}
534+
): number {
535+
const min = options.min ?? 0
536+
if (typeof value === 'number' && Number.isFinite(value) && value >= min) return value
537+
if (value === undefined || value === null || value === '') return fallback
538+
const parsed = Number(value)
539+
return Number.isFinite(parsed) && parsed >= min ? parsed : fallback
540+
}

‎apps/sim/lib/knowledge/documents/document-processor.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import {
1313
TokenChunker,
1414
} from '@/lib/chunkers'
1515
import type { ChunkingStrategy, StrategyOptions } from '@/lib/chunkers/types'
16-
import { env } from '@/lib/core/config/env'
16+
import { env, envNumber } from '@/lib/core/config/env'
1717
import { parseBuffer, parseFile } from '@/lib/file-parsers'
1818
import type { FileParseMetadata } from '@/lib/file-parsers/types'
1919
import { resolveParserExtension } from '@/lib/knowledge/documents/parser-extension'
@@ -30,7 +30,7 @@ const TIMEOUTS = {
3030
MISTRAL_OCR_API: 120000,
3131
} as const
3232

33-
const MAX_CONCURRENT_CHUNKS = env.KB_CONFIG_CHUNK_CONCURRENCY
33+
const MAX_CONCURRENT_CHUNKS = envNumber(env.KB_CONFIG_CHUNK_CONCURRENCY, 10)
3434

3535
type OCRResult = {
3636
success: boolean

‎apps/sim/lib/knowledge/documents/service.ts‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ import {
3030
import { recordUsage } from '@/lib/billing/core/usage-log'
3131
import { checkAndBillOverageThreshold } from '@/lib/billing/threshold-billing'
3232
import type { ChunkingStrategy, StrategyOptions } from '@/lib/chunkers/types'
33-
import { env } from '@/lib/core/config/env'
33+
import { env, envNumber } from '@/lib/core/config/env'
3434
import { getCostMultiplier, isTriggerDevEnabled } from '@/lib/core/config/feature-flags'
3535
import { processDocument } from '@/lib/knowledge/documents/document-processor'
3636
import type { DocumentSortField, SortOrder } from '@/lib/knowledge/documents/types'
@@ -54,12 +54,12 @@ import { calculateCost } from '@/providers/utils'
5454
const logger = createLogger('DocumentService')
5555

5656
const TIMEOUTS = {
57-
OVERALL_PROCESSING: (env.KB_CONFIG_MAX_DURATION || 600) * 1000,
57+
OVERALL_PROCESSING: envNumber(env.KB_CONFIG_MAX_DURATION, 600) * 1000,
5858
} as const
5959

6060
const LARGE_DOC_CONFIG = {
6161
MAX_CHUNKS_PER_BATCH: 500,
62-
MAX_EMBEDDING_BATCH: env.KB_CONFIG_BATCH_SIZE || 2000,
62+
MAX_EMBEDDING_BATCH: envNumber(env.KB_CONFIG_BATCH_SIZE, 2000),
6363
MAX_FILE_SIZE: 100 * 1024 * 1024,
6464
MAX_CHUNKS_PER_DOCUMENT: 100000,
6565
}
@@ -78,10 +78,11 @@ function withTimeout<T>(
7878
}
7979

8080
const PROCESSING_CONFIG = {
81-
maxConcurrentDocuments: Math.max(1, Math.floor((env.KB_CONFIG_CONCURRENCY_LIMIT || 20) / 5)) || 4,
82-
batchSize: Math.max(1, Math.floor((env.KB_CONFIG_BATCH_SIZE || 20) / 2)) || 10,
83-
delayBetweenBatches: (env.KB_CONFIG_DELAY_BETWEEN_BATCHES || 100) * 2,
84-
delayBetweenDocuments: (env.KB_CONFIG_DELAY_BETWEEN_DOCUMENTS || 50) * 2,
81+
maxConcurrentDocuments:
82+
Math.max(1, Math.floor(envNumber(env.KB_CONFIG_CONCURRENCY_LIMIT, 20) / 5)) || 4,
83+
batchSize: Math.max(1, Math.floor(envNumber(env.KB_CONFIG_BATCH_SIZE, 20) / 2)) || 10,
84+
delayBetweenBatches: envNumber(env.KB_CONFIG_DELAY_BETWEEN_BATCHES, 100) * 2,
85+
delayBetweenDocuments: envNumber(env.KB_CONFIG_DELAY_BETWEEN_DOCUMENTS, 50) * 2,
8586
}
8687

8788
export function getProcessingConfig() {

‎apps/sim/lib/knowledge/embeddings.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { createLogger } from '@sim/logger'
22
import { getBYOKKey } from '@/lib/api-key/byok'
33
import { getRotatingApiKey } from '@/lib/core/config/api-keys'
4-
import { env } from '@/lib/core/config/env'
4+
import { env, envNumber } from '@/lib/core/config/env'
55
import { isRetryableError, retryWithExponentialBackoff } from '@/lib/knowledge/documents/utils'
66
import {
77
DEFAULT_EMBEDDING_MODEL,
@@ -15,7 +15,7 @@ import { batchByTokenLimit, estimateTokenCount } from '@/lib/tokenization'
1515
const logger = createLogger('EmbeddingUtils')
1616

1717
const MAX_TOKENS_PER_REQUEST = 8000
18-
const MAX_CONCURRENT_BATCHES = env.KB_CONFIG_CONCURRENCY_LIMIT || 50
18+
const MAX_CONCURRENT_BATCHES = envNumber(env.KB_CONFIG_CONCURRENCY_LIMIT, 50)
1919
const EMBEDDING_REQUEST_TIMEOUT_MS = 60_000
2020

2121
export type { EmbeddingModelInfo } from '@/lib/knowledge/embedding-models'

‎apps/sim/lib/table/billing.ts‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import { createLogger } from '@sim/logger'
88
import { getHighestPrioritySubscription } from '@/lib/billing/core/subscription'
99
import { getPlanTypeForLimits } from '@/lib/billing/plan-helpers'
1010
import { getWorkspaceBilledAccountUserId } from '@/lib/workspaces/utils'
11-
import { type PlanName, TABLE_PLAN_LIMITS, type TablePlanLimits } from './constants'
11+
import { getTablePlanLimits, type PlanName, type TablePlanLimits } from './constants'
1212

1313
const logger = createLogger('TableBilling')
1414

@@ -22,18 +22,20 @@ const logger = createLogger('TableBilling')
2222
* @returns Table limits based on the workspace's billing plan
2323
*/
2424
export async function getWorkspaceTableLimits(workspaceId: string): Promise<TablePlanLimits> {
25+
const planLimits = getTablePlanLimits()
26+
2527
try {
2628
const billedAccountUserId = await getWorkspaceBilledAccountUserId(workspaceId)
2729

2830
if (!billedAccountUserId) {
2931
logger.warn('No billed account found for workspace, using free tier limits', { workspaceId })
30-
return TABLE_PLAN_LIMITS.free
32+
return planLimits.free
3133
}
3234

3335
const subscription = await getHighestPrioritySubscription(billedAccountUserId)
3436
const planName = getPlanTypeForLimits(subscription?.plan) as PlanName
3537

36-
const limits = TABLE_PLAN_LIMITS[planName] ?? TABLE_PLAN_LIMITS.free
38+
const limits = planLimits[planName] ?? planLimits.free
3739

3840
logger.info('Retrieved workspace table limits', {
3941
workspaceId,
@@ -48,7 +50,7 @@ export async function getWorkspaceTableLimits(workspaceId: string): Promise<Tabl
4850
workspaceId,
4951
error,
5052
})
51-
return TABLE_PLAN_LIMITS.free
53+
return planLimits.free
5254
}
5355
}
5456

‎apps/sim/lib/table/constants.ts‎

Lines changed: 50 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
* Limits and constants for user-defined tables.
33
*/
44

5+
import { env, envNumber } from '@/lib/core/config/env'
6+
57
export const TABLE_LIMITS = {
68
MAX_TABLES_PER_WORKSPACE: 100,
79
MAX_ROWS_PER_TABLE: 10000,
@@ -24,9 +26,11 @@ export const TABLE_LIMITS = {
2426
} as const
2527

2628
/**
27-
* Plan-based table limits.
29+
* Default plan-based table limits. Each value can be overridden via env vars
30+
* (see `getTablePlanLimits`) so self-hosted deployments can raise the free-tier
31+
* caps that apply when billing is disabled.
2832
*/
29-
export const TABLE_PLAN_LIMITS = {
33+
export const DEFAULT_TABLE_PLAN_LIMITS = {
3034
free: {
3135
maxTables: 3,
3236
maxRowsPerTable: 1000,
@@ -45,13 +49,56 @@ export const TABLE_PLAN_LIMITS = {
4549
},
4650
} as const
4751

48-
export type PlanName = keyof typeof TABLE_PLAN_LIMITS
52+
export type PlanName = keyof typeof DEFAULT_TABLE_PLAN_LIMITS
4953

5054
export interface TablePlanLimits {
5155
maxTables: number
5256
maxRowsPerTable: number
5357
}
5458

59+
export type TablePlanLimitsByPlan = Record<PlanName, TablePlanLimits>
60+
61+
/**
62+
* Returns plan-based table limits, applying env var overrides on top of the
63+
* defaults. When no override is set the value falls back to the hosted-default
64+
* constant so behavior is unchanged for the hosted product.
65+
*/
66+
export function getTablePlanLimits(): TablePlanLimitsByPlan {
67+
return {
68+
free: {
69+
maxTables: envNumber(env.FREE_TABLES_LIMIT, DEFAULT_TABLE_PLAN_LIMITS.free.maxTables),
70+
maxRowsPerTable: envNumber(
71+
env.FREE_TABLE_ROWS_LIMIT,
72+
DEFAULT_TABLE_PLAN_LIMITS.free.maxRowsPerTable
73+
),
74+
},
75+
pro: {
76+
maxTables: envNumber(env.PRO_TABLES_LIMIT, DEFAULT_TABLE_PLAN_LIMITS.pro.maxTables),
77+
maxRowsPerTable: envNumber(
78+
env.PRO_TABLE_ROWS_LIMIT,
79+
DEFAULT_TABLE_PLAN_LIMITS.pro.maxRowsPerTable
80+
),
81+
},
82+
team: {
83+
maxTables: envNumber(env.TEAM_TABLES_LIMIT, DEFAULT_TABLE_PLAN_LIMITS.team.maxTables),
84+
maxRowsPerTable: envNumber(
85+
env.TEAM_TABLE_ROWS_LIMIT,
86+
DEFAULT_TABLE_PLAN_LIMITS.team.maxRowsPerTable
87+
),
88+
},
89+
enterprise: {
90+
maxTables: envNumber(
91+
env.ENTERPRISE_TABLES_LIMIT,
92+
DEFAULT_TABLE_PLAN_LIMITS.enterprise.maxTables
93+
),
94+
maxRowsPerTable: envNumber(
95+
env.ENTERPRISE_TABLE_ROWS_LIMIT,
96+
DEFAULT_TABLE_PLAN_LIMITS.enterprise.maxRowsPerTable
97+
),
98+
},
99+
}
100+
}
101+
55102
export const COLUMN_TYPES = ['string', 'number', 'boolean', 'date', 'json'] as const
56103

57104
export const NAME_PATTERN = /^[a-z_][a-z0-9_]*$/i

0 commit comments

Comments
 (0)