fix(data-drains): correct Datadog size guard and Snowflake VARIANT limit

- Datadog payload guard now checks the uncompressed size against the 5 MB limit and the wire size against the 6 MB compressed limit, so gzip cannot smuggle an oversized body past the client-side check.
- Snowflake VARIANT limit is 16 MiB (16,777,216 bytes), not 16,000,000 bytes — small payloads between 16 MB and 16 MiB were being rejected unnecessarily.
- Drop the unused apiKey field on Datadog PostInput; the key is already embedded in the prepared request headers.

Author: waleedlatif1

apps/sim/lib/data-drains/destinations/datadog.ts

@@ -30,8 +30,13 @@ const MAX_ATTEMPTS = 4
 const BASE_BACKOFF_MS = 500
 const MAX_BACKOFF_MS = 30_000
 const PER_ATTEMPT_TIMEOUT_MS = 30_000
-/** Datadog v2 logs intake limits: 5 MB per request, 1 MB per entry, 1000 entries per request. */
-const MAX_REQUEST_BYTES = 5 * 1024 * 1024
+/**
+ * Datadog v2 logs intake limits: 5 MB uncompressed per request, 6 MB compressed
+ * (we enforce both since gzip can hide a too-large body), 1 MB per entry, 1000
+ * entries per request. https://docs.datadoghq.com/api/latest/logs/
+ */
+const MAX_UNCOMPRESSED_BYTES = 5 * 1024 * 1024
+const MAX_COMPRESSED_BYTES = 6 * 1024 * 1024
 const MAX_ENTRY_BYTES = 1024 * 1024
 const MAX_ENTRIES_PER_REQUEST = 1000
 /** Compress payloads above this threshold; gzip overhead isn't worth it on small bodies. */
@@ -127,21 +132,23 @@ function backoffWithJitter(attempt: number, retryAfterMs: number | null): number
 interface PreparedBody {
   body: Uint8Array | string
   headers: Record<string, string>
-  bytes: number
+  /** On-the-wire (post-gzip) size — what Datadog measures against its compressed limit. */
+  wireBytes: number
+  /** Uncompressed payload size — what Datadog measures against its uncompressed limit. */
+  rawBytes: number
 }
 
 interface PostInput {
   url: string
-  apiKey: string
   prepared: PreparedBody
   signal: AbortSignal
 }
 
 /**
  * Builds the request body and headers, applying gzip compression for payloads
  * above {@link GZIP_THRESHOLD_BYTES}. Returns the wire body (Buffer or string)
- * along with the headers describing it. The {@link MAX_REQUEST_BYTES} guard
- * applies to whatever is returned here (post-compression when applicable).
+ * along with the headers describing it. Both raw and wire sizes are returned
+ * so callers can enforce Datadog's 5 MB uncompressed / 6 MB compressed limits.
  */
 function buildRequestBody(payload: string, apiKey: string): PreparedBody {
   const headers: Record<string, string> = {
@@ -156,9 +163,9 @@ function buildRequestBody(payload: string, apiKey: string): PreparedBody {
     headers['Content-Encoding'] = 'gzip'
     // Re-wrap as a plain Uint8Array view so the fetch BodyInit overload matches.
     const view = new Uint8Array(compressed.buffer, compressed.byteOffset, compressed.byteLength)
-    return { body: view, headers, bytes: view.byteLength }
+    return { body: view, headers, wireBytes: view.byteLength, rawBytes }
   }
-  return { body: payload, headers, bytes: rawBytes }
+  return { body: payload, headers, wireBytes: rawBytes, rawBytes }
 }
 
 async function postWithRetries(input: PostInput): Promise<Response> {
@@ -217,7 +224,6 @@ export const datadogDestination: DrainDestination<
     ]
     await postWithRetries({
       url: buildEndpoint(config.site),
-      apiKey: credentials.apiKey,
       prepared: buildRequestBody(JSON.stringify(probe), credentials.apiKey),
       signal,
     })
@@ -244,22 +250,29 @@ export const datadogDestination: DrainDestination<
         }
         const payload = JSON.stringify(entries)
         const prepared = buildRequestBody(payload, credentials.apiKey)
-        if (prepared.bytes > MAX_REQUEST_BYTES) {
+        // Reject before sending so we surface a clean client-side error instead
+        // of letting Datadog return a confusing HTTP 413 after decompression.
+        if (prepared.rawBytes > MAX_UNCOMPRESSED_BYTES) {
+          throw new Error(
+            `Datadog payload is ${prepared.rawBytes} bytes uncompressed, exceeds the ${MAX_UNCOMPRESSED_BYTES}-byte per-request limit`
+          )
+        }
+        if (prepared.wireBytes > MAX_COMPRESSED_BYTES) {
           throw new Error(
-            `Datadog payload is ${prepared.bytes} bytes, exceeds the ${MAX_REQUEST_BYTES}-byte per-request limit`
+            `Datadog payload is ${prepared.wireBytes} bytes on the wire, exceeds the ${MAX_COMPRESSED_BYTES}-byte compressed per-request limit`
           )
         }
         const response = await postWithRetries({
           url,
-          apiKey: credentials.apiKey,
           prepared,
           signal,
         })
         const requestId = response.headers.get('dd-request-id') ?? null
         logger.debug('Datadog chunk delivered', {
           site: config.site,
           rows: entries.length,
-          bytes: prepared.bytes,
+          rawBytes: prepared.rawBytes,
+          wireBytes: prepared.wireBytes,
         })
         return {
           locator: requestId

apps/sim/lib/data-drains/destinations/snowflake.test.ts

@@ -179,9 +179,9 @@ describe('snowflakeDestination', () => {
     await session.close()
   })
 
-  it('throws a clear error when a binding exceeds the 16 MB VARIANT limit', async () => {
+  it('throws a clear error when a binding exceeds the 16 MiB VARIANT limit', async () => {
     const session = snowflakeDestination.openSession({ config, credentials })
-    const huge = `"${'a'.repeat(16_000_001)}"`
+    const huge = `"${'a'.repeat(16 * 1024 * 1024 + 1)}"`
     const body = Buffer.from(`${huge}\n`, 'utf8')
     await expect(
       session.deliver({

apps/sim/lib/data-drains/destinations/snowflake.ts

@@ -22,8 +22,13 @@ const POLL_DEADLINE_MS = 10 * 60_000
 const EXECUTE_MAX_ATTEMPTS = 3
 const EXECUTE_RETRY_BASE_DELAY_MS = 500
 const EXECUTE_RETRY_MAX_DELAY_MS = 5_000
-/** Snowflake VARIANT max value size is 16 MB. */
-const VARIANT_MAX_BYTES = 16_000_000
+/**
+ * Snowflake VARIANT max value size is 16 MiB (16,777,216 bytes) on accounts
+ * before the 2025_03 behavior change bundle, and 128 MB after it. We use the
+ * conservative pre-bundle limit so the same value works on every account.
+ * https://docs.snowflake.com/en/release-notes/bcr-bundles/2025_03/bcr-1942
+ */
+const VARIANT_MAX_BYTES = 16 * 1024 * 1024
 
 /**
  * Snowflake JWT `iss`/`sub` require the bare account identifier without any