From 2bc449f34da42352e9e444e11d75b8957702a4d4 Mon Sep 17 00:00:00 2001 From: Rafa Moreno Date: Fri, 6 Mar 2026 08:36:03 +0100 Subject: [PATCH 1/6] feat: add 429 header-based retries for ingest --- README.md | 17 +++ src/api/api.test.ts | 250 ++++++++++++++++++++++++++++++++++++++++++++ src/api/api.ts | 153 ++++++++++++++++++++++++--- src/client/types.ts | 18 ++++ src/index.ts | 1 + 5 files changed, 426 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index adba6d2..c68cddd 100644 --- a/README.md +++ b/README.md @@ -247,6 +247,23 @@ await api.ingest("events", { pathname: "/home", }); +// Ingest with retries for rate limiting (HTTP 429 only, disabled by default). +// Retries are attempted only when Retry-After / X-RateLimit-Reset is present. +await api.ingest( + "events", + { + timestamp: "2024-01-15 10:31:00", + event_name: "button_click", + pathname: "/pricing", + }, + { + wait: true, + retry: { + maxRetries: 3, + }, + } +); + // Import rows from URL/file await api.appendDatasource("events", { url: "https://example.com/events.csv", diff --git a/src/api/api.test.ts b/src/api/api.test.ts index 7a317bb..05bc74a 100644 --- a/src/api/api.test.ts +++ b/src/api/api.test.ts @@ -207,6 +207,256 @@ describe("TinybirdApi", () => { ).rejects.toThrow("Date values are not supported in ingest payloads"); }); + it("does not retry ingest on 503", async () => { + let attempts = 0; + + server.use( + http.post(`${BASE_URL}/v0/events`, () => { + attempts += 1; + return new HttpResponse("Service unavailable", { status: 503 }); + }) + ); + + const api = createTinybirdApi({ + baseUrl: BASE_URL, + token: "p.default-token", + }); + + await expect( + api.ingest( + "events", + { timestamp: "2024-01-01 00:00:00" }, + { + retry: { + maxRetries: 1, + }, + } + ) + ).rejects.toMatchObject({ + name: "TinybirdApiError", + statusCode: 503, + }); + expect(attempts).toBe(1); + }); + + it("retries ingest on 429 with retry-after header and succeeds", async () => { + let attempts = 0; + + server.use( + http.post(`${BASE_URL}/v0/events`, () => { + attempts += 1; + if (attempts === 1) { + return new HttpResponse("Rate limited", { + status: 429, + headers: { + "Retry-After": "0", + }, + }); + } + + return HttpResponse.json({ + successful_rows: 1, + quarantined_rows: 0, + }); + }) + ); + + const api = createTinybirdApi({ + baseUrl: BASE_URL, + token: "p.default-token", + }); + + const result = await api.ingest( + "events", + { timestamp: "2024-01-01 00:00:00" }, + { + retry: { + maxRetries: 1, + }, + } + ); + + expect(result).toEqual({ successful_rows: 1, quarantined_rows: 0 }); + expect(attempts).toBe(2); + }); + + it("does not retry 429 when rate-limit delay headers are missing", async () => { + let attempts = 0; + + server.use( + http.post(`${BASE_URL}/v0/events`, () => { + attempts += 1; + return new HttpResponse("Rate limited", { status: 429 }); + }) + ); + + const api = createTinybirdApi({ + baseUrl: BASE_URL, + token: "p.default-token", + }); + + await expect( + api.ingest( + "events", + { timestamp: "2024-01-01 00:00:00" }, + { + retry: { + maxRetries: 3, + }, + } + ) + ).rejects.toMatchObject({ + name: "TinybirdApiError", + statusCode: 429, + }); + expect(attempts).toBe(1); + }); + + it("does not retry ingest on non-retryable status by default", async () => { + let attempts = 0; + + server.use( + http.post(`${BASE_URL}/v0/events`, () => { + attempts += 1; + return HttpResponse.json({ error: "Invalid payload" }, { status: 400 }); + }) + ); + + const api = createTinybirdApi({ + baseUrl: BASE_URL, + token: "p.default-token", + }); + + await expect( + api.ingest( + "events", + { timestamp: "2024-01-01 00:00:00" }, + { + retry: { + maxRetries: 3, + }, + } + ) + ).rejects.toMatchObject({ + name: "TinybirdApiError", + statusCode: 400, + }); + + expect(attempts).toBe(1); + }); + + it("stops retrying ingest after maxRetries on 429", async () => { + let attempts = 0; + + server.use( + http.post(`${BASE_URL}/v0/events`, () => { + attempts += 1; + return new HttpResponse("Rate limited", { + status: 429, + headers: { + "Retry-After": "0", + }, + }); + }) + ); + + const api = createTinybirdApi({ + baseUrl: BASE_URL, + token: "p.default-token", + }); + + await expect( + api.ingest( + "events", + { timestamp: "2024-01-01 00:00:00" }, + { + retry: { + maxRetries: 2, + }, + } + ) + ).rejects.toMatchObject({ + name: "TinybirdApiError", + statusCode: 429, + }); + + expect(attempts).toBe(3); + }); + + it("does not retry 500 even when wait is false", async () => { + let attempts = 0; + + server.use( + http.post(`${BASE_URL}/v0/events`, () => { + attempts += 1; + return new HttpResponse("Internal error", { status: 500 }); + }) + ); + + const api = createTinybirdApi({ + baseUrl: BASE_URL, + token: "p.default-token", + }); + + await expect( + api.ingest( + "events", + { timestamp: "2024-01-01 00:00:00" }, + { + wait: false, + retry: { + maxRetries: 3, + }, + } + ) + ).rejects.toMatchObject({ + name: "TinybirdApiError", + statusCode: 500, + }); + + expect(attempts).toBe(1); + }); + + it("does not retry ingest on transient network errors", async () => { + let fetchAttempts = 0; + + server.use( + http.post(`${BASE_URL}/v0/events`, () => { + return HttpResponse.json({ + successful_rows: 1, + quarantined_rows: 0, + }); + }) + ); + + const flakyFetch: typeof fetch = async (input, init) => { + fetchAttempts += 1; + if (fetchAttempts === 1) { + throw new TypeError("fetch failed"); + } + return fetch(input, init); + }; + + const api = createTinybirdApi({ + baseUrl: BASE_URL, + token: "p.default-token", + fetch: flakyFetch, + }); + + await expect( + api.ingest( + "events", + { timestamp: "2024-01-01 00:00:00" }, + { + retry: { + maxRetries: 1, + }, + } + ) + ).rejects.toThrow("fetch failed"); + expect(fetchAttempts).toBe(1); + }); + it("executes raw SQL via tinybirdApi.sql", async () => { let rawSql: string | null = null; let contentType: string | null = null; diff --git a/src/api/api.ts b/src/api/api.ts index e78f73b..f973b80 100644 --- a/src/api/api.ts +++ b/src/api/api.ts @@ -14,6 +14,7 @@ import type { } from "../client/types.js"; const DEFAULT_TIMEOUT = 30000; +const DEFAULT_INGEST_RETRY_MAX_RETRIES = 2; /** * Public, decoupled Tinybird API wrapper configuration @@ -64,6 +65,10 @@ export interface TinybirdApiTruncateOptions extends TruncateOptions { token?: string; } +interface NormalizedIngestRetryOptions { + maxRetries: number; +} + /** * Scope definition for token creation APIs */ @@ -279,22 +284,47 @@ export class TinybirdApi { const ndjson = events .map((event) => JSON.stringify(this.serializeEvent(event))) .join("\n"); + const signal = this.createAbortSignal(options.timeout, options.signal); + const retryOptions = this.resolveIngestRetryOptions(options.retry); + let retryCount = 0; + + while (true) { + let response: Response; + + try { + response = await this.request(url.toString(), { + method: "POST", + token: options.token, + headers: { + "Content-Type": "application/x-ndjson", + }, + body: ndjson, + signal, + }); + } catch (error) { + throw error; + } - const response = await this.request(url.toString(), { - method: "POST", - token: options.token, - headers: { - "Content-Type": "application/x-ndjson", - }, - body: ndjson, - signal: this.createAbortSignal(options.timeout, options.signal), - }); + if (response.ok) { + return (await response.json()) as IngestResult; + } - if (!response.ok) { - await this.handleErrorResponse(response); - } + if ( + !retryOptions || + retryCount >= retryOptions.maxRetries || + !this.shouldRetryIngestStatus(response.status) + ) { + await this.handleErrorResponse(response); + } - return (await response.json()) as IngestResult; + const retryDelay = this.resolveRetryDelayFromHeaders(response); + if (retryDelay === undefined) { + await this.handleErrorResponse(response); + } + + await this.sleep(retryDelay!, signal); + retryCount += 1; + } } /** @@ -575,6 +605,103 @@ export class TinybirdApi { return AbortSignal.any([timeoutSignal, existingSignal]); } + private resolveIngestRetryOptions( + retry: TinybirdApiIngestOptions["retry"] + ): NormalizedIngestRetryOptions | undefined { + if (!retry) { + return undefined; + } + + return { + maxRetries: retry.maxRetries ?? DEFAULT_INGEST_RETRY_MAX_RETRIES, + }; + } + + private shouldRetryIngestStatus(statusCode: number): boolean { + return statusCode === 429; + } + + private resolveRetryDelayFromHeaders(response: Response): number | undefined { + const retryAfter = response.headers.get("retry-after"); + const retryAfterDelay = this.parseRetryAfterDelayMs(retryAfter); + if (retryAfterDelay !== undefined) { + return retryAfterDelay; + } + + const rateLimitReset = response.headers.get("x-ratelimit-reset"); + const rateLimitResetDelay = this.parseRateLimitResetDelayMs(rateLimitReset); + if (rateLimitResetDelay !== undefined) { + return rateLimitResetDelay; + } + return undefined; + } + + private parseRetryAfterDelayMs(value: string | null): number | undefined { + if (!value) { + return undefined; + } + + const trimmed = value.trim(); + const seconds = Number(trimmed); + if (Number.isFinite(seconds)) { + return Math.max(0, Math.floor(seconds * 1000)); + } + + const retryDateMs = Date.parse(trimmed); + if (Number.isNaN(retryDateMs)) { + return undefined; + } + + return Math.max(0, retryDateMs - Date.now()); + } + + private parseRateLimitResetDelayMs(value: string | null): number | undefined { + if (!value) { + return undefined; + } + + const numericValue = Number(value.trim()); + if (!Number.isFinite(numericValue)) { + return undefined; + } + + return Math.max(0, Math.floor(numericValue * 1000)); + } + + private async sleep(delayMs: number, signal?: AbortSignal): Promise { + if (delayMs <= 0) { + return; + } + + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + cleanup(); + resolve(); + }, delayMs); + + const onAbort = () => { + cleanup(); + reject(signal?.reason ?? new DOMException("The operation was aborted.", "AbortError")); + }; + + const cleanup = () => { + clearTimeout(timer); + signal?.removeEventListener("abort", onAbort); + }; + + if (!signal) { + return; + } + + if (signal.aborted) { + onAbort(); + return; + } + + signal.addEventListener("abort", onAbort, { once: true }); + }); + } + private serializeEvent( event: Record ): Record { diff --git a/src/client/types.ts b/src/client/types.ts index 49171e4..785c86d 100644 --- a/src/client/types.ts +++ b/src/client/types.ts @@ -152,6 +152,24 @@ export interface IngestOptions { signal?: AbortSignal; /** Wait for the ingestion to complete before returning */ wait?: boolean; + /** + * Retry strategy for transient ingest failures. + * Set to false to disable retries (default behavior). + */ + retry?: IngestRetryOptions | false; +} + +/** + * Retry strategy for ingest requests. + * Retries apply only to HTTP 429 responses and only when the response includes + * Retry-After or X-RateLimit-Reset headers. + */ +export interface IngestRetryOptions { + /** + * Number of retry attempts after the first request. + * Example: 2 means at most 3 total attempts. + */ + maxRetries?: number; } /** diff --git a/src/index.ts b/src/index.ts index 1712103..c506e76 100644 --- a/src/index.ts +++ b/src/index.ts @@ -249,6 +249,7 @@ export type { IngestResult, QueryOptions, IngestOptions, + IngestRetryOptions, TruncateOptions, TruncateResult, ColumnMeta, From 08a7fe8eae4a88cb163adfa218d178c25db24b8b Mon Sep 17 00:00:00 2001 From: Rafa Moreno Date: Fri, 6 Mar 2026 10:01:57 +0100 Subject: [PATCH 2/6] fix: drain 429 response body before ingest retry --- src/api/api.test.ts | 58 +++++++++++++++++++++++++++++++++++++++++++++ src/api/api.ts | 17 +++++++++++++ 2 files changed, 75 insertions(+) diff --git a/src/api/api.test.ts b/src/api/api.test.ts index 05bc74a..96d9011 100644 --- a/src/api/api.test.ts +++ b/src/api/api.test.ts @@ -280,6 +280,64 @@ describe("TinybirdApi", () => { expect(attempts).toBe(2); }); + it("drains retryable 429 response body before retrying", async () => { + let attempts = 0; + let firstResponse: Response | undefined; + + const customFetch: typeof fetch = async () => { + attempts += 1; + if (attempts === 1) { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("rate limited")); + controller.close(); + }, + }); + + firstResponse = new Response(stream, { + status: 429, + headers: { + "Retry-After": "0", + }, + }); + return firstResponse; + } + + return new Response( + JSON.stringify({ + successful_rows: 1, + quarantined_rows: 0, + }), + { + status: 200, + headers: { + "Content-Type": "application/json", + }, + } + ); + }; + + const api = createTinybirdApi({ + baseUrl: BASE_URL, + token: "p.default-token", + fetch: customFetch, + }); + + const result = await api.ingest( + "events", + { timestamp: "2024-01-01 00:00:00" }, + { + retry: { + maxRetries: 1, + }, + } + ); + + expect(result).toEqual({ successful_rows: 1, quarantined_rows: 0 }); + expect(attempts).toBe(2); + expect(firstResponse?.bodyUsed).toBe(true); + }); + it("does not retry 429 when rate-limit delay headers are missing", async () => { let attempts = 0; diff --git a/src/api/api.ts b/src/api/api.ts index f973b80..0922977 100644 --- a/src/api/api.ts +++ b/src/api/api.ts @@ -322,6 +322,7 @@ export class TinybirdApi { await this.handleErrorResponse(response); } + await this.discardResponseBody(response); await this.sleep(retryDelay!, signal); retryCount += 1; } @@ -668,6 +669,22 @@ export class TinybirdApi { return Math.max(0, Math.floor(numericValue * 1000)); } + private async discardResponseBody(response: Response): Promise { + if (response.bodyUsed || !response.body) { + return; + } + + try { + await response.arrayBuffer(); + } catch { + try { + await response.body.cancel(); + } catch { + // Best effort cleanup only; never mask retry/error flow. + } + } + } + private async sleep(delayMs: number, signal?: AbortSignal): Promise { if (delayMs <= 0) { return; From dc42db66c71563f79af746f002475b026f9803f5 Mon Sep 17 00:00:00 2001 From: Rafa Moreno Date: Fri, 6 Mar 2026 10:34:42 +0100 Subject: [PATCH 3/6] feat: add configurable 503 ingest retries with backoff --- README.md | 10 +++- src/api/api.test.ts | 119 ++++++++++++++++++++++++++++++++++++++++++ src/api/api.ts | 122 ++++++++++++++++++++++++++++++++++++++------ src/client/types.ts | 28 ++++++++-- src/index.ts | 1 + 5 files changed, 259 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index c68cddd..89011a2 100644 --- a/README.md +++ b/README.md @@ -247,8 +247,9 @@ await api.ingest("events", { pathname: "/home", }); -// Ingest with retries for rate limiting (HTTP 429 only, disabled by default). -// Retries are attempted only when Retry-After / X-RateLimit-Reset is present. +// Ingest retry behavior (disabled by default): +// - 429 retries use Retry-After / X-RateLimit-Reset headers. +// - 503 retries are optional and use exponential backoff (wait=true only). await api.ingest( "events", { @@ -260,6 +261,11 @@ await api.ingest( wait: true, retry: { maxRetries: 3, + retry503: { + maxRetries: 2, + baseDelayMs: 250, + maxDelayMs: 3000, + }, }, } ); diff --git a/src/api/api.test.ts b/src/api/api.test.ts index 96d9011..7954314 100644 --- a/src/api/api.test.ts +++ b/src/api/api.test.ts @@ -239,6 +239,48 @@ describe("TinybirdApi", () => { expect(attempts).toBe(1); }); + it("retries ingest on 503 with exponential backoff when configured", async () => { + let attempts = 0; + + server.use( + http.post(`${BASE_URL}/v0/events`, () => { + attempts += 1; + if (attempts === 1) { + return new HttpResponse("Service unavailable", { status: 503 }); + } + + return HttpResponse.json({ + successful_rows: 1, + quarantined_rows: 0, + }); + }) + ); + + const api = createTinybirdApi({ + baseUrl: BASE_URL, + token: "p.default-token", + }); + + const result = await api.ingest( + "events", + { timestamp: "2024-01-01 00:00:00" }, + { + retry: { + maxRetries: 1, + retry503: { + maxRetries: 1, + baseDelayMs: 0, + maxDelayMs: 0, + jitter: false, + }, + }, + } + ); + + expect(result).toEqual({ successful_rows: 1, quarantined_rows: 0 }); + expect(attempts).toBe(2); + }); + it("retries ingest on 429 with retry-after header and succeeds", async () => { let attempts = 0; @@ -441,6 +483,83 @@ describe("TinybirdApi", () => { expect(attempts).toBe(3); }); + it("stops retrying ingest after maxRetries on 503 when configured", async () => { + let attempts = 0; + + server.use( + http.post(`${BASE_URL}/v0/events`, () => { + attempts += 1; + return new HttpResponse("Service unavailable", { status: 503 }); + }) + ); + + const api = createTinybirdApi({ + baseUrl: BASE_URL, + token: "p.default-token", + }); + + await expect( + api.ingest( + "events", + { timestamp: "2024-01-01 00:00:00" }, + { + retry: { + retry503: { + maxRetries: 2, + baseDelayMs: 0, + maxDelayMs: 0, + jitter: false, + }, + }, + } + ) + ).rejects.toMatchObject({ + name: "TinybirdApiError", + statusCode: 503, + }); + + expect(attempts).toBe(3); + }); + + it("does not retry ingest on 503 when wait is false", async () => { + let attempts = 0; + + server.use( + http.post(`${BASE_URL}/v0/events`, () => { + attempts += 1; + return new HttpResponse("Service unavailable", { status: 503 }); + }) + ); + + const api = createTinybirdApi({ + baseUrl: BASE_URL, + token: "p.default-token", + }); + + await expect( + api.ingest( + "events", + { timestamp: "2024-01-01 00:00:00" }, + { + wait: false, + retry: { + retry503: { + maxRetries: 3, + baseDelayMs: 0, + maxDelayMs: 0, + jitter: false, + }, + }, + } + ) + ).rejects.toMatchObject({ + name: "TinybirdApiError", + statusCode: 503, + }); + + expect(attempts).toBe(1); + }); + it("does not retry 500 even when wait is false", async () => { let attempts = 0; diff --git a/src/api/api.ts b/src/api/api.ts index 0922977..b1c4e5d 100644 --- a/src/api/api.ts +++ b/src/api/api.ts @@ -5,6 +5,7 @@ import type { DeleteOptions, DeleteResult, IngestOptions, + IngestRetry503Options, IngestResult, QueryOptions, QueryResult, @@ -15,6 +16,9 @@ import type { const DEFAULT_TIMEOUT = 30000; const DEFAULT_INGEST_RETRY_MAX_RETRIES = 2; +const DEFAULT_INGEST_RETRY_503_MAX_RETRIES = 2; +const DEFAULT_INGEST_RETRY_503_BASE_DELAY_MS = 200; +const DEFAULT_INGEST_RETRY_503_MAX_DELAY_MS = 3000; /** * Public, decoupled Tinybird API wrapper configuration @@ -67,6 +71,14 @@ export interface TinybirdApiTruncateOptions extends TruncateOptions { interface NormalizedIngestRetryOptions { maxRetries: number; + retry503?: NormalizedIngestRetry503Options; +} + +interface NormalizedIngestRetry503Options { + maxRetries: number; + baseDelayMs: number; + maxDelayMs: number; + jitter: boolean; } /** @@ -286,7 +298,9 @@ export class TinybirdApi { .join("\n"); const signal = this.createAbortSignal(options.timeout, options.signal); const retryOptions = this.resolveIngestRetryOptions(options.retry); - let retryCount = 0; + const waitEnabled = options.wait !== false; + let retry429Count = 0; + let retry503Count = 0; while (true) { let response: Response; @@ -309,22 +323,28 @@ export class TinybirdApi { return (await response.json()) as IngestResult; } - if ( - !retryOptions || - retryCount >= retryOptions.maxRetries || - !this.shouldRetryIngestStatus(response.status) - ) { - await this.handleErrorResponse(response); + const retry429Delay = this.resolveRetry429Delay(response, retryOptions, retry429Count); + if (retry429Delay !== undefined) { + await this.discardResponseBody(response); + await this.sleep(retry429Delay, signal); + retry429Count += 1; + continue; } - const retryDelay = this.resolveRetryDelayFromHeaders(response); - if (retryDelay === undefined) { - await this.handleErrorResponse(response); + const retry503Delay = this.resolveRetry503Delay( + response, + retryOptions, + retry503Count, + waitEnabled + ); + if (retry503Delay !== undefined) { + await this.discardResponseBody(response); + await this.sleep(retry503Delay, signal); + retry503Count += 1; + continue; } - await this.discardResponseBody(response); - await this.sleep(retryDelay!, signal); - retryCount += 1; + await this.handleErrorResponse(response); } } @@ -613,13 +633,67 @@ export class TinybirdApi { return undefined; } - return { + const normalized: NormalizedIngestRetryOptions = { maxRetries: retry.maxRetries ?? DEFAULT_INGEST_RETRY_MAX_RETRIES, }; + + if (retry.retry503) { + normalized.retry503 = this.resolveIngestRetry503Options(retry.retry503); + } + + return normalized; } - private shouldRetryIngestStatus(statusCode: number): boolean { - return statusCode === 429; + private resolveIngestRetry503Options( + retry503: IngestRetry503Options + ): NormalizedIngestRetry503Options { + return { + maxRetries: retry503.maxRetries ?? DEFAULT_INGEST_RETRY_503_MAX_RETRIES, + baseDelayMs: retry503.baseDelayMs ?? DEFAULT_INGEST_RETRY_503_BASE_DELAY_MS, + maxDelayMs: retry503.maxDelayMs ?? DEFAULT_INGEST_RETRY_503_MAX_DELAY_MS, + jitter: retry503.jitter ?? true, + }; + } + + private resolveRetry429Delay( + response: Response, + retryOptions: NormalizedIngestRetryOptions | undefined, + retry429Count: number + ): number | undefined { + if (!retryOptions) { + return undefined; + } + + if (response.status !== 429) { + return undefined; + } + + if (retry429Count >= retryOptions.maxRetries) { + return undefined; + } + + return this.resolveRetryDelayFromHeaders(response); + } + + private resolveRetry503Delay( + response: Response, + retryOptions: NormalizedIngestRetryOptions | undefined, + retry503Count: number, + waitEnabled: boolean + ): number | undefined { + if (!retryOptions?.retry503 || !waitEnabled) { + return undefined; + } + + if (response.status !== 503) { + return undefined; + } + + if (retry503Count >= retryOptions.retry503.maxRetries) { + return undefined; + } + + return this.calculateRetry503DelayMs(retryOptions.retry503, retry503Count); } private resolveRetryDelayFromHeaders(response: Response): number | undefined { @@ -669,6 +743,22 @@ export class TinybirdApi { return Math.max(0, Math.floor(numericValue * 1000)); } + private calculateRetry503DelayMs( + retry503: NormalizedIngestRetry503Options, + retryCount: number + ): number { + const exponentialDelay = Math.min( + retry503.maxDelayMs, + retry503.baseDelayMs * 2 ** retryCount + ); + + if (!retry503.jitter) { + return exponentialDelay; + } + + return Math.floor(Math.random() * (exponentialDelay + 1)); + } + private async discardResponseBody(response: Response): Promise { if (response.bodyUsed || !response.body) { return; diff --git a/src/client/types.ts b/src/client/types.ts index 785c86d..0650cff 100644 --- a/src/client/types.ts +++ b/src/client/types.ts @@ -161,15 +161,37 @@ export interface IngestOptions { /** * Retry strategy for ingest requests. - * Retries apply only to HTTP 429 responses and only when the response includes - * Retry-After or X-RateLimit-Reset headers. + * - HTTP 429: retries require Retry-After or X-RateLimit-Reset headers. + * - HTTP 503: retries are optional and configurable via `retry503`. */ export interface IngestRetryOptions { /** - * Number of retry attempts after the first request. + * Number of retry attempts after the first request for HTTP 429. * Example: 2 means at most 3 total attempts. */ maxRetries?: number; + /** + * Optional retry strategy for HTTP 503 responses (wait=true only). + * Set to false or leave undefined to disable 503 retries. + */ + retry503?: IngestRetry503Options | false; +} + +/** + * Retry strategy for HTTP 503 ingest responses. + */ +export interface IngestRetry503Options { + /** + * Number of retry attempts after the first 503 response. + * Example: 2 means at most 3 total attempts. + */ + maxRetries?: number; + /** Base delay in milliseconds for exponential backoff (default: 200) */ + baseDelayMs?: number; + /** Maximum delay in milliseconds for exponential backoff (default: 3000) */ + maxDelayMs?: number; + /** Add random jitter to delay to avoid synchronized retries (default: true) */ + jitter?: boolean; } /** diff --git a/src/index.ts b/src/index.ts index c506e76..5abf854 100644 --- a/src/index.ts +++ b/src/index.ts @@ -250,6 +250,7 @@ export type { QueryOptions, IngestOptions, IngestRetryOptions, + IngestRetry503Options, TruncateOptions, TruncateResult, ColumnMeta, From 258ec2d54676591aebc97db04258675187c4dbd0 Mon Sep 17 00:00:00 2001 From: Rafa Moreno Date: Fri, 6 Mar 2026 10:38:04 +0100 Subject: [PATCH 4/6] refactor: remove jitter from ingest retry backoff --- src/api/api.test.ts | 3 --- src/api/api.ts | 10 +--------- src/client/types.ts | 2 -- 3 files changed, 1 insertion(+), 14 deletions(-) diff --git a/src/api/api.test.ts b/src/api/api.test.ts index 7954314..ac4278a 100644 --- a/src/api/api.test.ts +++ b/src/api/api.test.ts @@ -271,7 +271,6 @@ describe("TinybirdApi", () => { maxRetries: 1, baseDelayMs: 0, maxDelayMs: 0, - jitter: false, }, }, } @@ -508,7 +507,6 @@ describe("TinybirdApi", () => { maxRetries: 2, baseDelayMs: 0, maxDelayMs: 0, - jitter: false, }, }, } @@ -547,7 +545,6 @@ describe("TinybirdApi", () => { maxRetries: 3, baseDelayMs: 0, maxDelayMs: 0, - jitter: false, }, }, } diff --git a/src/api/api.ts b/src/api/api.ts index b1c4e5d..1f9b3b7 100644 --- a/src/api/api.ts +++ b/src/api/api.ts @@ -78,7 +78,6 @@ interface NormalizedIngestRetry503Options { maxRetries: number; baseDelayMs: number; maxDelayMs: number; - jitter: boolean; } /** @@ -651,7 +650,6 @@ export class TinybirdApi { maxRetries: retry503.maxRetries ?? DEFAULT_INGEST_RETRY_503_MAX_RETRIES, baseDelayMs: retry503.baseDelayMs ?? DEFAULT_INGEST_RETRY_503_BASE_DELAY_MS, maxDelayMs: retry503.maxDelayMs ?? DEFAULT_INGEST_RETRY_503_MAX_DELAY_MS, - jitter: retry503.jitter ?? true, }; } @@ -747,16 +745,10 @@ export class TinybirdApi { retry503: NormalizedIngestRetry503Options, retryCount: number ): number { - const exponentialDelay = Math.min( + return Math.min( retry503.maxDelayMs, retry503.baseDelayMs * 2 ** retryCount ); - - if (!retry503.jitter) { - return exponentialDelay; - } - - return Math.floor(Math.random() * (exponentialDelay + 1)); } private async discardResponseBody(response: Response): Promise { diff --git a/src/client/types.ts b/src/client/types.ts index 0650cff..338f5b5 100644 --- a/src/client/types.ts +++ b/src/client/types.ts @@ -190,8 +190,6 @@ export interface IngestRetry503Options { baseDelayMs?: number; /** Maximum delay in milliseconds for exponential backoff (default: 3000) */ maxDelayMs?: number; - /** Add random jitter to delay to avoid synchronized retries (default: true) */ - jitter?: boolean; } /** From 4821adeed0da0c2d9e2c4e06fdd6fa12927c5366 Mon Sep 17 00:00:00 2001 From: Rafa Moreno Date: Fri, 6 Mar 2026 10:44:42 +0100 Subject: [PATCH 5/6] refactor: use global ingest maxRetries for 503 retries --- README.md | 8 +---- src/api/api.test.ts | 73 ++++++++++++++++++--------------------------- src/api/api.ts | 66 ++++++++++------------------------------ src/client/types.ts | 24 ++------------- src/index.ts | 1 - 5 files changed, 47 insertions(+), 125 deletions(-) diff --git a/README.md b/README.md index 89011a2..0194d18 100644 --- a/README.md +++ b/README.md @@ -249,7 +249,7 @@ await api.ingest("events", { // Ingest retry behavior (disabled by default): // - 429 retries use Retry-After / X-RateLimit-Reset headers. -// - 503 retries are optional and use exponential backoff (wait=true only). +// - 503 retries use SDK default exponential backoff. await api.ingest( "events", { @@ -258,14 +258,8 @@ await api.ingest( pathname: "/pricing", }, { - wait: true, retry: { maxRetries: 3, - retry503: { - maxRetries: 2, - baseDelayMs: 250, - maxDelayMs: 3000, - }, }, } ); diff --git a/src/api/api.test.ts b/src/api/api.test.ts index ac4278a..9f783c7 100644 --- a/src/api/api.test.ts +++ b/src/api/api.test.ts @@ -207,7 +207,7 @@ describe("TinybirdApi", () => { ).rejects.toThrow("Date values are not supported in ingest payloads"); }); - it("does not retry ingest on 503", async () => { + it("does not retry ingest on 503 when retry is disabled", async () => { let attempts = 0; server.use( @@ -223,15 +223,7 @@ describe("TinybirdApi", () => { }); await expect( - api.ingest( - "events", - { timestamp: "2024-01-01 00:00:00" }, - { - retry: { - maxRetries: 1, - }, - } - ) + api.ingest("events", { timestamp: "2024-01-01 00:00:00" }) ).rejects.toMatchObject({ name: "TinybirdApiError", statusCode: 503, @@ -239,7 +231,7 @@ describe("TinybirdApi", () => { expect(attempts).toBe(1); }); - it("retries ingest on 503 with exponential backoff when configured", async () => { + it("retries ingest on 503 with exponential backoff", async () => { let attempts = 0; server.use( @@ -267,11 +259,6 @@ describe("TinybirdApi", () => { { retry: { maxRetries: 1, - retry503: { - maxRetries: 1, - baseDelayMs: 0, - maxDelayMs: 0, - }, }, } ); @@ -482,7 +469,7 @@ describe("TinybirdApi", () => { expect(attempts).toBe(3); }); - it("stops retrying ingest after maxRetries on 503 when configured", async () => { + it("stops retrying ingest after maxRetries on 503", async () => { let attempts = 0; server.use( @@ -503,11 +490,7 @@ describe("TinybirdApi", () => { { timestamp: "2024-01-01 00:00:00" }, { retry: { - retry503: { - maxRetries: 2, - baseDelayMs: 0, - maxDelayMs: 0, - }, + maxRetries: 2, }, } ) @@ -519,13 +502,20 @@ describe("TinybirdApi", () => { expect(attempts).toBe(3); }); - it("does not retry ingest on 503 when wait is false", async () => { + it("retries ingest on 503 when wait is false", async () => { let attempts = 0; server.use( http.post(`${BASE_URL}/v0/events`, () => { attempts += 1; - return new HttpResponse("Service unavailable", { status: 503 }); + if (attempts === 1) { + return new HttpResponse("Service unavailable", { status: 503 }); + } + + return HttpResponse.json({ + successful_rows: 1, + quarantined_rows: 0, + }); }) ); @@ -534,27 +524,22 @@ describe("TinybirdApi", () => { token: "p.default-token", }); - await expect( - api.ingest( - "events", - { timestamp: "2024-01-01 00:00:00" }, - { - wait: false, - retry: { - retry503: { - maxRetries: 3, - baseDelayMs: 0, - maxDelayMs: 0, - }, - }, - } - ) - ).rejects.toMatchObject({ - name: "TinybirdApiError", - statusCode: 503, - }); + const result = await api.ingest( + "events", + { timestamp: "2024-01-01 00:00:00" }, + { + wait: false, + retry: { + maxRetries: 1, + }, + } + ); - expect(attempts).toBe(1); + expect(result).toEqual({ + successful_rows: 1, + quarantined_rows: 0, + }); + expect(attempts).toBe(2); }); it("does not retry 500 even when wait is false", async () => { diff --git a/src/api/api.ts b/src/api/api.ts index 1f9b3b7..e8f7cb3 100644 --- a/src/api/api.ts +++ b/src/api/api.ts @@ -5,7 +5,6 @@ import type { DeleteOptions, DeleteResult, IngestOptions, - IngestRetry503Options, IngestResult, QueryOptions, QueryResult, @@ -16,7 +15,6 @@ import type { const DEFAULT_TIMEOUT = 30000; const DEFAULT_INGEST_RETRY_MAX_RETRIES = 2; -const DEFAULT_INGEST_RETRY_503_MAX_RETRIES = 2; const DEFAULT_INGEST_RETRY_503_BASE_DELAY_MS = 200; const DEFAULT_INGEST_RETRY_503_MAX_DELAY_MS = 3000; @@ -71,13 +69,6 @@ export interface TinybirdApiTruncateOptions extends TruncateOptions { interface NormalizedIngestRetryOptions { maxRetries: number; - retry503?: NormalizedIngestRetry503Options; -} - -interface NormalizedIngestRetry503Options { - maxRetries: number; - baseDelayMs: number; - maxDelayMs: number; } /** @@ -297,9 +288,7 @@ export class TinybirdApi { .join("\n"); const signal = this.createAbortSignal(options.timeout, options.signal); const retryOptions = this.resolveIngestRetryOptions(options.retry); - const waitEnabled = options.wait !== false; - let retry429Count = 0; - let retry503Count = 0; + let retryCount = 0; while (true) { let response: Response; @@ -322,24 +311,19 @@ export class TinybirdApi { return (await response.json()) as IngestResult; } - const retry429Delay = this.resolveRetry429Delay(response, retryOptions, retry429Count); + const retry429Delay = this.resolveRetry429Delay(response, retryOptions, retryCount); if (retry429Delay !== undefined) { await this.discardResponseBody(response); await this.sleep(retry429Delay, signal); - retry429Count += 1; + retryCount += 1; continue; } - const retry503Delay = this.resolveRetry503Delay( - response, - retryOptions, - retry503Count, - waitEnabled - ); + const retry503Delay = this.resolveRetry503Delay(response, retryOptions, retryCount); if (retry503Delay !== undefined) { await this.discardResponseBody(response); await this.sleep(retry503Delay, signal); - retry503Count += 1; + retryCount += 1; continue; } @@ -632,31 +616,15 @@ export class TinybirdApi { return undefined; } - const normalized: NormalizedIngestRetryOptions = { - maxRetries: retry.maxRetries ?? DEFAULT_INGEST_RETRY_MAX_RETRIES, - }; - - if (retry.retry503) { - normalized.retry503 = this.resolveIngestRetry503Options(retry.retry503); - } - - return normalized; - } - - private resolveIngestRetry503Options( - retry503: IngestRetry503Options - ): NormalizedIngestRetry503Options { return { - maxRetries: retry503.maxRetries ?? DEFAULT_INGEST_RETRY_503_MAX_RETRIES, - baseDelayMs: retry503.baseDelayMs ?? DEFAULT_INGEST_RETRY_503_BASE_DELAY_MS, - maxDelayMs: retry503.maxDelayMs ?? DEFAULT_INGEST_RETRY_503_MAX_DELAY_MS, + maxRetries: retry.maxRetries ?? DEFAULT_INGEST_RETRY_MAX_RETRIES, }; } private resolveRetry429Delay( response: Response, retryOptions: NormalizedIngestRetryOptions | undefined, - retry429Count: number + retryCount: number ): number | undefined { if (!retryOptions) { return undefined; @@ -666,7 +634,7 @@ export class TinybirdApi { return undefined; } - if (retry429Count >= retryOptions.maxRetries) { + if (retryCount >= retryOptions.maxRetries) { return undefined; } @@ -676,10 +644,9 @@ export class TinybirdApi { private resolveRetry503Delay( response: Response, retryOptions: NormalizedIngestRetryOptions | undefined, - retry503Count: number, - waitEnabled: boolean + retryCount: number ): number | undefined { - if (!retryOptions?.retry503 || !waitEnabled) { + if (!retryOptions) { return undefined; } @@ -687,11 +654,11 @@ export class TinybirdApi { return undefined; } - if (retry503Count >= retryOptions.retry503.maxRetries) { + if (retryCount >= retryOptions.maxRetries) { return undefined; } - return this.calculateRetry503DelayMs(retryOptions.retry503, retry503Count); + return this.calculateRetry503DelayMs(retryCount); } private resolveRetryDelayFromHeaders(response: Response): number | undefined { @@ -741,13 +708,10 @@ export class TinybirdApi { return Math.max(0, Math.floor(numericValue * 1000)); } - private calculateRetry503DelayMs( - retry503: NormalizedIngestRetry503Options, - retryCount: number - ): number { + private calculateRetry503DelayMs(retryCount: number): number { return Math.min( - retry503.maxDelayMs, - retry503.baseDelayMs * 2 ** retryCount + DEFAULT_INGEST_RETRY_503_MAX_DELAY_MS, + DEFAULT_INGEST_RETRY_503_BASE_DELAY_MS * 2 ** retryCount ); } diff --git a/src/client/types.ts b/src/client/types.ts index 338f5b5..f1adc9b 100644 --- a/src/client/types.ts +++ b/src/client/types.ts @@ -162,34 +162,14 @@ export interface IngestOptions { /** * Retry strategy for ingest requests. * - HTTP 429: retries require Retry-After or X-RateLimit-Reset headers. - * - HTTP 503: retries are optional and configurable via `retry503`. + * - HTTP 503: retries use SDK default exponential backoff. */ export interface IngestRetryOptions { /** - * Number of retry attempts after the first request for HTTP 429. + * Number of retry attempts after the first request. * Example: 2 means at most 3 total attempts. */ maxRetries?: number; - /** - * Optional retry strategy for HTTP 503 responses (wait=true only). - * Set to false or leave undefined to disable 503 retries. - */ - retry503?: IngestRetry503Options | false; -} - -/** - * Retry strategy for HTTP 503 ingest responses. - */ -export interface IngestRetry503Options { - /** - * Number of retry attempts after the first 503 response. - * Example: 2 means at most 3 total attempts. - */ - maxRetries?: number; - /** Base delay in milliseconds for exponential backoff (default: 200) */ - baseDelayMs?: number; - /** Maximum delay in milliseconds for exponential backoff (default: 3000) */ - maxDelayMs?: number; } /** diff --git a/src/index.ts b/src/index.ts index 5abf854..c506e76 100644 --- a/src/index.ts +++ b/src/index.ts @@ -250,7 +250,6 @@ export type { QueryOptions, IngestOptions, IngestRetryOptions, - IngestRetry503Options, TruncateOptions, TruncateResult, ColumnMeta, From 5bd9fa83a2eb3466434355490884a7b46582ac9c Mon Sep 17 00:00:00 2001 From: Rafa Moreno Date: Fri, 6 Mar 2026 10:52:37 +0100 Subject: [PATCH 6/6] refactor: move ingest retries to top-level maxRetries --- README.md | 4 +--- package.json | 2 +- src/api/api.test.ts | 40 ++++++++++------------------------------ src/api/api.ts | 39 ++++++++++++++++++--------------------- src/client/types.ts | 15 +-------------- src/index.ts | 1 - 6 files changed, 31 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index 0194d18..f5ff299 100644 --- a/README.md +++ b/README.md @@ -258,9 +258,7 @@ await api.ingest( pathname: "/pricing", }, { - retry: { - maxRetries: 3, - }, + maxRetries: 3, } ); diff --git a/package.json b/package.json index f38d7fb..6e0832e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tinybirdco/sdk", - "version": "0.0.57", + "version": "0.0.58", "description": "TypeScript SDK for Tinybird Forward - define datasources and pipes as TypeScript", "type": "module", "main": "./dist/index.js", diff --git a/src/api/api.test.ts b/src/api/api.test.ts index 9f783c7..2480cec 100644 --- a/src/api/api.test.ts +++ b/src/api/api.test.ts @@ -257,9 +257,7 @@ describe("TinybirdApi", () => { "events", { timestamp: "2024-01-01 00:00:00" }, { - retry: { - maxRetries: 1, - }, + maxRetries: 1, } ); @@ -298,9 +296,7 @@ describe("TinybirdApi", () => { "events", { timestamp: "2024-01-01 00:00:00" }, { - retry: { - maxRetries: 1, - }, + maxRetries: 1, } ); @@ -355,9 +351,7 @@ describe("TinybirdApi", () => { "events", { timestamp: "2024-01-01 00:00:00" }, { - retry: { - maxRetries: 1, - }, + maxRetries: 1, } ); @@ -386,9 +380,7 @@ describe("TinybirdApi", () => { "events", { timestamp: "2024-01-01 00:00:00" }, { - retry: { - maxRetries: 3, - }, + maxRetries: 3, } ) ).rejects.toMatchObject({ @@ -418,9 +410,7 @@ describe("TinybirdApi", () => { "events", { timestamp: "2024-01-01 00:00:00" }, { - retry: { - maxRetries: 3, - }, + maxRetries: 3, } ) ).rejects.toMatchObject({ @@ -456,9 +446,7 @@ describe("TinybirdApi", () => { "events", { timestamp: "2024-01-01 00:00:00" }, { - retry: { - maxRetries: 2, - }, + maxRetries: 2, } ) ).rejects.toMatchObject({ @@ -489,9 +477,7 @@ describe("TinybirdApi", () => { "events", { timestamp: "2024-01-01 00:00:00" }, { - retry: { - maxRetries: 2, - }, + maxRetries: 2, } ) ).rejects.toMatchObject({ @@ -529,9 +515,7 @@ describe("TinybirdApi", () => { { timestamp: "2024-01-01 00:00:00" }, { wait: false, - retry: { - maxRetries: 1, - }, + maxRetries: 1, } ); @@ -563,9 +547,7 @@ describe("TinybirdApi", () => { { timestamp: "2024-01-01 00:00:00" }, { wait: false, - retry: { - maxRetries: 3, - }, + maxRetries: 3, } ) ).rejects.toMatchObject({ @@ -607,9 +589,7 @@ describe("TinybirdApi", () => { "events", { timestamp: "2024-01-01 00:00:00" }, { - retry: { - maxRetries: 1, - }, + maxRetries: 1, } ) ).rejects.toThrow("fetch failed"); diff --git a/src/api/api.ts b/src/api/api.ts index e8f7cb3..dab93fe 100644 --- a/src/api/api.ts +++ b/src/api/api.ts @@ -14,7 +14,6 @@ import type { } from "../client/types.js"; const DEFAULT_TIMEOUT = 30000; -const DEFAULT_INGEST_RETRY_MAX_RETRIES = 2; const DEFAULT_INGEST_RETRY_503_BASE_DELAY_MS = 200; const DEFAULT_INGEST_RETRY_503_MAX_DELAY_MS = 3000; @@ -67,10 +66,6 @@ export interface TinybirdApiTruncateOptions extends TruncateOptions { token?: string; } -interface NormalizedIngestRetryOptions { - maxRetries: number; -} - /** * Scope definition for token creation APIs */ @@ -287,7 +282,7 @@ export class TinybirdApi { .map((event) => JSON.stringify(this.serializeEvent(event))) .join("\n"); const signal = this.createAbortSignal(options.timeout, options.signal); - const retryOptions = this.resolveIngestRetryOptions(options.retry); + const maxRetries = this.resolveIngestMaxRetries(options.maxRetries); let retryCount = 0; while (true) { @@ -311,7 +306,7 @@ export class TinybirdApi { return (await response.json()) as IngestResult; } - const retry429Delay = this.resolveRetry429Delay(response, retryOptions, retryCount); + const retry429Delay = this.resolveRetry429Delay(response, maxRetries, retryCount); if (retry429Delay !== undefined) { await this.discardResponseBody(response); await this.sleep(retry429Delay, signal); @@ -319,7 +314,7 @@ export class TinybirdApi { continue; } - const retry503Delay = this.resolveRetry503Delay(response, retryOptions, retryCount); + const retry503Delay = this.resolveRetry503Delay(response, maxRetries, retryCount); if (retry503Delay !== undefined) { await this.discardResponseBody(response); await this.sleep(retry503Delay, signal); @@ -609,24 +604,26 @@ export class TinybirdApi { return AbortSignal.any([timeoutSignal, existingSignal]); } - private resolveIngestRetryOptions( - retry: TinybirdApiIngestOptions["retry"] - ): NormalizedIngestRetryOptions | undefined { - if (!retry) { + private resolveIngestMaxRetries( + maxRetries: TinybirdApiIngestOptions["maxRetries"] + ): number | undefined { + if (maxRetries === undefined) { return undefined; } - return { - maxRetries: retry.maxRetries ?? DEFAULT_INGEST_RETRY_MAX_RETRIES, - }; + if (!Number.isFinite(maxRetries)) { + throw new Error("'maxRetries' must be a finite number"); + } + + return Math.max(0, Math.floor(maxRetries)); } private resolveRetry429Delay( response: Response, - retryOptions: NormalizedIngestRetryOptions | undefined, + maxRetries: number | undefined, retryCount: number ): number | undefined { - if (!retryOptions) { + if (maxRetries === undefined) { return undefined; } @@ -634,7 +631,7 @@ export class TinybirdApi { return undefined; } - if (retryCount >= retryOptions.maxRetries) { + if (retryCount >= maxRetries) { return undefined; } @@ -643,10 +640,10 @@ export class TinybirdApi { private resolveRetry503Delay( response: Response, - retryOptions: NormalizedIngestRetryOptions | undefined, + maxRetries: number | undefined, retryCount: number ): number | undefined { - if (!retryOptions) { + if (maxRetries === undefined) { return undefined; } @@ -654,7 +651,7 @@ export class TinybirdApi { return undefined; } - if (retryCount >= retryOptions.maxRetries) { + if (retryCount >= maxRetries) { return undefined; } diff --git a/src/client/types.ts b/src/client/types.ts index f1adc9b..e7b093b 100644 --- a/src/client/types.ts +++ b/src/client/types.ts @@ -152,22 +152,9 @@ export interface IngestOptions { signal?: AbortSignal; /** Wait for the ingestion to complete before returning */ wait?: boolean; - /** - * Retry strategy for transient ingest failures. - * Set to false to disable retries (default behavior). - */ - retry?: IngestRetryOptions | false; -} - -/** - * Retry strategy for ingest requests. - * - HTTP 429: retries require Retry-After or X-RateLimit-Reset headers. - * - HTTP 503: retries use SDK default exponential backoff. - */ -export interface IngestRetryOptions { /** * Number of retry attempts after the first request. - * Example: 2 means at most 3 total attempts. + * Retries are disabled by default when undefined. */ maxRetries?: number; } diff --git a/src/index.ts b/src/index.ts index c506e76..1712103 100644 --- a/src/index.ts +++ b/src/index.ts @@ -249,7 +249,6 @@ export type { IngestResult, QueryOptions, IngestOptions, - IngestRetryOptions, TruncateOptions, TruncateResult, ColumnMeta,