From c39242cc0d4480bee1221ed7be806bd23b7b63f9 Mon Sep 17 00:00:00 2001 From: Stefinie Fernando Date: Fri, 24 Jul 2026 15:18:29 +0530 Subject: [PATCH 1/2] feat: implement exponential backoff for OpenChoreo catalog sync retries Signed-off-by: Stefinie Fernando --- .changeset/catalog-sync-retry-backoff.md | 11 ++ app-config.production.yaml | 4 + app-config.yaml | 1 + .../src/ClientCredentialsProvider.test.ts | 103 +++++++++++ .../src/ClientCredentialsProvider.ts | 164 +++++++++++------- .../src/module.ts | 11 +- .../provider/OpenChoreoEntityProvider.test.ts | 96 ++++++++++ .../src/provider/OpenChoreoEntityProvider.ts | 85 ++++++++- .../src/utils/openChoreoApiClient.ts | 14 +- plugins/openchoreo-common/config.d.ts | 7 + 10 files changed, 425 insertions(+), 71 deletions(-) create mode 100644 .changeset/catalog-sync-retry-backoff.md create mode 100644 packages/openchoreo-auth/src/ClientCredentialsProvider.test.ts diff --git a/.changeset/catalog-sync-retry-backoff.md b/.changeset/catalog-sync-retry-backoff.md new file mode 100644 index 000000000..71c3b879b --- /dev/null +++ b/.changeset/catalog-sync-retry-backoff.md @@ -0,0 +1,11 @@ +--- +'@openchoreo/backstage-plugin-catalog-backend-module': patch +'@openchoreo/openchoreo-auth': patch +'@openchoreo/backstage-plugin-common': patch +--- + +Recover the OpenChoreo catalog sync quickly after a failed run instead of waiting a full sync interval. Previously a failure (e.g. the IdP/API being briefly unreachable during a cluster cold start) aborted the periodic full sync, leaving the catalog empty until the next scheduled run (default 5 minutes) later. + +The entity provider now retries a failed sync with capped exponential backoff — starting at `openchoreo.schedule.retryInterval` (new, default 30s) and doubling on each consecutive failure up to `schedule.frequency` — so it recovers within ~30s of the backend returning, backs off gracefully during a sustained outage instead of hammering, and never stops reconciling (the catalog self-heals whenever the backend comes back). + +Client-credentials token acquisition also retries transient failures (network errors and 5xx, but not 4xx) with exponential backoff, and the catalog API client now fails fast when a service token cannot be obtained rather than silently issuing unauthenticated requests that are guaranteed to 401. diff --git a/app-config.production.yaml b/app-config.production.yaml index de02d25e1..5a8b1fe2c 100644 --- a/app-config.production.yaml +++ b/app-config.production.yaml @@ -202,6 +202,10 @@ openchoreo: schedule: frequency: ${OPENCHOREO_CATALOG_SYNC_FREQUENCY} + # Seconds between retries while a sync is failing; backs off exponentially + # up to `frequency`. Environment variable: OPENCHOREO_CATALOG_SYNC_RETRY_INTERVAL + # (seconds). When unset it resolves to undefined and the code default (30) applies. + retryInterval: ${OPENCHOREO_CATALOG_SYNC_RETRY_INTERVAL} timeout: 120 # seconds for timeout (default: 120) # Whether to wire the OpenChoreo event-driven catalog sync. Mirrors diff --git a/app-config.yaml b/app-config.yaml index 3dd91d51b..2f5d34a8b 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -185,6 +185,7 @@ openchoreo: schedule: frequency: 300 # seconds between runs (default: 300, the periodic full sync acts as a safety net behind the event-forwarder's real-time deltas) + retryInterval: 30 # seconds between retries while a sync is failing (default: 30); recovers fast from cold-start dependency outages without waiting a full frequency timeout: 120 # seconds for timeout (default: 120) # Observability tuning (optional). diff --git a/packages/openchoreo-auth/src/ClientCredentialsProvider.test.ts b/packages/openchoreo-auth/src/ClientCredentialsProvider.test.ts new file mode 100644 index 000000000..48eed3b84 --- /dev/null +++ b/packages/openchoreo-auth/src/ClientCredentialsProvider.test.ts @@ -0,0 +1,103 @@ +import { ClientCredentialsProvider } from './ClientCredentialsProvider'; +import { OpenChoreoAuthConfig } from './types'; + +const authConfig: OpenChoreoAuthConfig = { + clientId: 'test-client', + clientSecret: 'test-secret', + tokenUrl: 'http://idp.test/oauth2/token', + scopes: [], +}; + +function mkLogger() { + return { + info: jest.fn(), + debug: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + child: jest.fn().mockReturnThis(), + } as any; +} + +// `access_token` is intentionally not a real JWT; decodeJwt throws and the +// provider falls back to `expires_in`, which is all these tests need. +const okResponse = () => ({ + ok: true, + status: 200, + statusText: 'OK', + json: async () => ({ access_token: 'access-token', expires_in: 3600 }), + text: async () => '', +}); + +const errorResponse = (status: number) => ({ + ok: false, + status, + statusText: 'Error', + json: async () => ({}), + text: async () => 'error body', +}); + +describe('ClientCredentialsProvider', () => { + const realFetch = global.fetch; + + afterEach(() => { + global.fetch = realFetch; + jest.useRealTimers(); + jest.clearAllMocks(); + }); + + function mockFetch(...outcomes: Array) { + const fn = jest.fn(); + for (const outcome of outcomes) { + if (outcome instanceof Error) { + fn.mockRejectedValueOnce(outcome); + } else { + fn.mockResolvedValueOnce(outcome); + } + } + global.fetch = fn as any; + return fn; + } + + it('returns the token on first success without retrying', async () => { + const fetchMock = mockFetch(okResponse()); + const provider = new ClientCredentialsProvider(authConfig, mkLogger()); + + await expect(provider.getToken()).resolves.toBe('access-token'); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('retries a network error and then succeeds', async () => { + jest.useFakeTimers(); + const fetchMock = mockFetch(new Error('fetch failed'), okResponse()); + const provider = new ClientCredentialsProvider(authConfig, mkLogger()); + + const tokenPromise = provider.getToken(); + await jest.advanceTimersByTimeAsync(1000); // first backoff + await expect(tokenPromise).resolves.toBe('access-token'); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it('does not retry a 4xx response', async () => { + const fetchMock = mockFetch(errorResponse(401)); + const provider = new ClientCredentialsProvider(authConfig, mkLogger()); + + await expect(provider.getToken()).rejects.toThrow(/401/); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('retries a 5xx response up to the max attempts, then throws', async () => { + jest.useFakeTimers(); + const fetchMock = mockFetch( + errorResponse(503), + errorResponse(503), + errorResponse(503), + ); + const provider = new ClientCredentialsProvider(authConfig, mkLogger()); + + const tokenPromise = provider.getToken(); + tokenPromise.catch(() => {}); // avoid unhandled rejection while timers advance + await jest.advanceTimersByTimeAsync(1000 + 2000); // both backoffs + await expect(tokenPromise).rejects.toThrow(/503/); + expect(fetchMock).toHaveBeenCalledTimes(3); + }); +}); diff --git a/packages/openchoreo-auth/src/ClientCredentialsProvider.ts b/packages/openchoreo-auth/src/ClientCredentialsProvider.ts index f1dec9d7a..bb6d9b7da 100644 --- a/packages/openchoreo-auth/src/ClientCredentialsProvider.ts +++ b/packages/openchoreo-auth/src/ClientCredentialsProvider.ts @@ -8,6 +8,25 @@ import { OpenChoreoAuthConfig, CachedToken, TokenResponse } from './types'; */ const TOKEN_EXPIRY_BUFFER_MS = 60 * 1000; +/** Number of token-fetch attempts before giving up. */ +const TOKEN_FETCH_MAX_ATTEMPTS = 3; +/** Base backoff between token-fetch retries (doubles each attempt). */ +const TOKEN_FETCH_BASE_DELAY_MS = 1000; + +const delay = (ms: number): Promise => + new Promise(resolve => setTimeout(resolve, ms)); + +/** + * Error from a single token request, tagged with whether retrying could help. + * Network failures and 5xx are retryable; 4xx (e.g. bad credentials) are not. + */ +class TokenRequestError extends Error { + constructor(message: string, readonly retryable: boolean) { + super(message); + this.name = 'TokenRequestError'; + } +} + /** * Provides OAuth2 access tokens using the client credentials grant flow. * Used for background tasks that don't have a user context. @@ -61,9 +80,41 @@ export class ClientCredentialsProvider { } /** - * Fetches a new access token from the OAuth2 token endpoint. + * Fetches a new access token, retrying transient failures with exponential + * backoff. Only network errors and 5xx responses are retried; 4xx (e.g. bad + * credentials) fail immediately since retrying cannot help. */ private async fetchNewToken(): Promise { + let lastError: unknown; + for (let attempt = 1; attempt <= TOKEN_FETCH_MAX_ATTEMPTS; attempt++) { + try { + return await this.requestToken(); + } catch (error) { + lastError = error; + const retryable = + !(error instanceof TokenRequestError) || error.retryable; + if (!retryable || attempt === TOKEN_FETCH_MAX_ATTEMPTS) { + break; + } + const backoffMs = TOKEN_FETCH_BASE_DELAY_MS * 2 ** (attempt - 1); + this.logger.warn( + `Client credentials token fetch attempt ${attempt}/${TOKEN_FETCH_MAX_ATTEMPTS} failed; retrying in ${backoffMs}ms: ${error}`, + ); + await delay(backoffMs); + } + } + this.logger.error( + 'Failed to fetch client credentials token', + lastError as Error, + ); + throw lastError; + } + + /** + * Performs a single token request against the OAuth2 token endpoint. + * Throws a {@link TokenRequestError} on non-ok responses. + */ + private async requestToken(): Promise { this.logger.debug('Fetching new client credentials token'); const { clientId, clientSecret, tokenUrl, scopes } = this.config; @@ -78,74 +129,67 @@ export class ClientCredentialsProvider { params.set('scope', scopes.join(' ')); } - try { - const response = await fetch(tokenUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/x-www-form-urlencoded', - }, - body: params.toString(), - }); - - if (!response.ok) { - const errorText = await response.text(); - throw new Error( - `Token request failed: ${response.status} ${response.statusText} - ${errorText}`, - ); - } + const response = await fetch(tokenUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body: params.toString(), + }); - const tokenResponse: TokenResponse = await response.json(); + if (!response.ok) { + const errorText = await response.text(); + throw new TokenRequestError( + `Token request failed: ${response.status} ${response.statusText} - ${errorText}`, + response.status >= 500, + ); + } - // Decode JWT to extract the actual expiration time from the exp claim - // This is more accurate than using expires_in, which doesn't account for - // time elapsed between token issuance and response receipt - let expiresAt: number; - try { - const decodedToken = decodeJwt(tokenResponse.access_token); - if (decodedToken.exp) { - // exp claim is in seconds, convert to milliseconds - expiresAt = decodedToken.exp * 1000; - this.logger.debug( - `Using JWT exp claim for expiry: ${expiresAt} (${new Date( - expiresAt, - ).toISOString()})`, - ); - } else { - // Fallback to expires_in if exp claim is missing - expiresAt = Date.now() + tokenResponse.expires_in * 1000; - this.logger.debug( - `JWT exp claim missing, falling back to expires_in calculation: ${expiresAt}`, - ); - } - } catch (error) { - // Fallback to expires_in if JWT decoding fails + const tokenResponse: TokenResponse = await response.json(); + + // Decode JWT to extract the actual expiration time from the exp claim + // This is more accurate than using expires_in, which doesn't account for + // time elapsed between token issuance and response receipt + let expiresAt: number; + try { + const decodedToken = decodeJwt(tokenResponse.access_token); + if (decodedToken.exp) { + // exp claim is in seconds, convert to milliseconds + expiresAt = decodedToken.exp * 1000; + this.logger.debug( + `Using JWT exp claim for expiry: ${expiresAt} (${new Date( + expiresAt, + ).toISOString()})`, + ); + } else { + // Fallback to expires_in if exp claim is missing expiresAt = Date.now() + tokenResponse.expires_in * 1000; this.logger.debug( - `Failed to decode JWT token, falling back to expires_in calculation: ${error}`, + `JWT exp claim missing, falling back to expires_in calculation: ${expiresAt}`, ); } - - // Cache the token - this.cachedToken = { - accessToken: tokenResponse.access_token, - expiresAt, - }; - - const expiresInSeconds = Math.round((expiresAt - Date.now()) / 1000); - this.logger.debug( - `Successfully obtained client credentials token, expires in ${expiresInSeconds}s at ${new Date( - expiresAt, - ).toISOString()}`, - ); - - return tokenResponse.access_token; } catch (error) { - this.logger.error( - 'Failed to fetch client credentials token', - error as Error, + // Fallback to expires_in if JWT decoding fails + expiresAt = Date.now() + tokenResponse.expires_in * 1000; + this.logger.debug( + `Failed to decode JWT token, falling back to expires_in calculation: ${error}`, ); - throw error; } + + // Cache the token + this.cachedToken = { + accessToken: tokenResponse.access_token, + expiresAt, + }; + + const expiresInSeconds = Math.round((expiresAt - Date.now()) / 1000); + this.logger.debug( + `Successfully obtained client credentials token, expires in ${expiresInSeconds}s at ${new Date( + expiresAt, + ).toISOString()}`, + ); + + return tokenResponse.access_token; } /** diff --git a/plugins/catalog-backend-module-openchoreo/src/module.ts b/plugins/catalog-backend-module-openchoreo/src/module.ts index b67478ae2..e982f4e0a 100644 --- a/plugins/catalog-backend-module-openchoreo/src/module.ts +++ b/plugins/catalog-backend-module-openchoreo/src/module.ts @@ -96,8 +96,13 @@ export const catalogModuleOpenchoreo = createBackendModule({ urlReader, }) { const openchoreoConfig = config.getOptionalConfig('openchoreo'); - const frequency = - openchoreoConfig?.getOptionalNumber('schedule.frequency') ?? 300; + // The scheduled task fires at this fast retry cadence; the provider + // throttles so a healthy sync still only runs every + // `schedule.frequency` seconds (which the provider reads itself), + // but a failed run (e.g. dependency unreachable at cold start) + // retries within `retryInterval` instead of waiting a full frequency. + const retryInterval = + openchoreoConfig?.getOptionalNumber('schedule.retryInterval') ?? 30; const timeout = openchoreoConfig?.getOptionalNumber('schedule.timeout') ?? 120; // Whether to wire up the event-driven sync. Defaults to true so @@ -111,7 +116,7 @@ export const catalogModuleOpenchoreo = createBackendModule({ openchoreoConfig?.getOptionalBoolean('events.enabled') ?? true; const taskRunner = scheduler.createScheduledTaskRunner({ - frequency: { seconds: frequency }, + frequency: { seconds: retryInterval }, timeout: { seconds: timeout }, }); diff --git a/plugins/catalog-backend-module-openchoreo/src/provider/OpenChoreoEntityProvider.test.ts b/plugins/catalog-backend-module-openchoreo/src/provider/OpenChoreoEntityProvider.test.ts index a7533e005..6c24e1127 100644 --- a/plugins/catalog-backend-module-openchoreo/src/provider/OpenChoreoEntityProvider.test.ts +++ b/plugins/catalog-backend-module-openchoreo/src/provider/OpenChoreoEntityProvider.test.ts @@ -1189,4 +1189,100 @@ describe('OpenChoreoEntityProvider', () => { expect((comp?.spec as any).owner).toBe('group:default/test-owner'); }); }); + + describe('adaptive sync cadence', () => { + afterEach(() => { + jest.useRealTimers(); + }); + + // frequency = 300s (normal cadence), retryInterval = 30s (backoff base). + function scheduleConfig() { + return new ConfigReader({ + openchoreo: { + baseUrl: 'http://test:8080', + defaultOwner: 'test-owner', + componentTypes: { + mappings: [{ pattern: 'Service', pageVariant: 'service' }], + }, + schedule: { frequency: 300, retryInterval: 30 }, + }, + }); + } + + async function connectProvider(config = scheduleConfig()) { + const provider = new OpenChoreoEntityProvider( + taskRunner as any, + mkLogger(), + config, + undefined, // no token service + ); + await provider.connect(mockConnection as any); + return provider; + } + + it('syncs on the first tick, then no-ops while healthy and not yet due', async () => { + // Every endpoint returns an empty list → the run succeeds (applies an + // empty full mutation). A second tick soon after is skipped because a + // sync just succeeded and the normal frequency has not elapsed. + mockGET.mockResolvedValue(okData({ items: [] })); + + await connectProvider(); + await taskRunner.runTask(); + await taskRunner.runTask(); + + expect(mockConnection.applyMutation).toHaveBeenCalledTimes(1); + }); + + it('retries failures with capped exponential backoff and resets on success', async () => { + jest.useFakeTimers(); + jest.setSystemTime(0); + + // Count how many full-sync attempts actually reach the API (each run + // hits /api/v1/namespaces exactly once). `failing` flips to false to + // simulate the backend recovering. + let namespacesCalls = 0; + let failing = true; + mockGET.mockImplementation((path: string) => { + if (path === '/api/v1/namespaces') { + namespacesCalls += 1; + return Promise.resolve( + failing ? errorData(500) : okData({ items: [] }), + ); + } + return Promise.resolve(okData({ items: [] })); + }); + + await connectProvider(); + + // t=0: first attempt fails → next retry at +30s. + await taskRunner.runTask(); + expect(namespacesCalls).toBe(1); + + // t=0 again: still inside the backoff window → no-op. + await taskRunner.runTask(); + expect(namespacesCalls).toBe(1); + + // t=30s: retry due → 2nd failure → next retry at +60s (t=90s). + jest.setSystemTime(30_000); + await taskRunner.runTask(); + expect(namespacesCalls).toBe(2); + + // t=60s: backoff widened to 60s, so still not due → no-op. + jest.setSystemTime(60_000); + await taskRunner.runTask(); + expect(namespacesCalls).toBe(2); + + // t=90s: retry due, backend recovered → success, no mutation missed. + jest.setSystemTime(90_000); + failing = false; + await taskRunner.runTask(); + expect(namespacesCalls).toBe(3); + expect(mockConnection.applyMutation).toHaveBeenCalledTimes(1); + + // After success the failure state resets: an immediate tick no-ops + // (healthy, frequency not elapsed) rather than retrying fast. + await taskRunner.runTask(); + expect(namespacesCalls).toBe(3); + }); + }); }); diff --git a/plugins/catalog-backend-module-openchoreo/src/provider/OpenChoreoEntityProvider.ts b/plugins/catalog-backend-module-openchoreo/src/provider/OpenChoreoEntityProvider.ts index 456f284aa..b6b7de6a7 100644 --- a/plugins/catalog-backend-module-openchoreo/src/provider/OpenChoreoEntityProvider.ts +++ b/plugins/catalog-backend-module-openchoreo/src/provider/OpenChoreoEntityProvider.ts @@ -134,6 +134,16 @@ export class OpenChoreoEntityProvider implements EntityProvider { /** Resolves custom templates; undefined disables the feature (no UrlReader). */ private readonly remoteTemplateFetcher?: RemoteTemplateFetcher; + // Adaptive-cadence bookkeeping. The scheduled task fires every + // `retryInterval` seconds; these let a healthy sync stay on the normal + // `frequency` while a failed run retries with exponential backoff. + private lastSuccessAtMs?: number; + private consecutiveFailures = 0; + // Earliest time the next retry may run while failing (0 = retry now). + private nextRetryAtMs = 0; + private readonly syncFrequencyMs: number; + private readonly retryBaseMs: number; + constructor( taskRunner: SchedulerServiceTaskRunner, logger: LoggerService, @@ -147,6 +157,16 @@ export class OpenChoreoEntityProvider implements EntityProvider { this.taskRunner = taskRunner; this.logger = logger; this.baseUrl = config.getString('openchoreo.baseUrl'); + // Normal (healthy) full-sync cadence. Must match module.ts's + // `schedule.frequency`; the task itself fires more often (retryInterval). + this.syncFrequencyMs = + (config.getOptionalNumber('openchoreo.schedule.frequency') ?? 300) * 1000; + // Base delay for the first retry after a failure; doubles each + // consecutive failure, capped at syncFrequencyMs. Must match module.ts's + // `schedule.retryInterval` (the tick cadence). + this.retryBaseMs = + (config.getOptionalNumber('openchoreo.schedule.retryInterval') ?? 30) * + 1000; this.tokenService = tokenService; this.events = events; // Default owner for built-in Backstage entities (Domain, System, Component, API) @@ -303,11 +323,67 @@ export class OpenChoreoEntityProvider implements EntityProvider { await this.taskRunner.run({ id: this.getProviderName(), fn: async () => { - await this.run(); + await this.syncTick(); }, }); } + /** + * Scheduled tick. The task fires every `retryInterval` seconds, but a full + * sync only runs when it is actually due: + * - healthy: every `syncFrequencyMs` (the normal cadence); + * - failing: after an exponential backoff (retryBase, x2 per consecutive + * failure, capped at syncFrequencyMs) — so a cold-start failure recovers + * within ~retryBase, while a sustained outage backs off to the normal + * cadence instead of hammering. It never gives up: the loop keeps + * reconciling so the catalog self-heals whenever the backend returns. + * Ticks that are neither due nor past the backoff window are cheap no-ops. + */ + private async syncTick(): Promise { + if (!this.connection) { + throw new Error('Connection not initialized'); + } + const now = Date.now(); + if (this.consecutiveFailures > 0) { + // Failing: honor the exponential backoff window. (Don't fall back to + // the "normal cadence" branch — while failing, backoff is the only + // gate, otherwise a never-yet-succeeded provider would run every tick.) + if (now < this.nextRetryAtMs) { + return; + } + } else { + // Healthy: run only on the normal cadence. The first-ever run + // (no prior success recorded) is always due. + const dueForNormalSync = + this.lastSuccessAtMs === undefined || + now - this.lastSuccessAtMs >= this.syncFrequencyMs; + if (!dueForNormalSync) { + return; + } + } + + const succeeded = await this.runNew(); + if (succeeded) { + this.consecutiveFailures = 0; + this.nextRetryAtMs = 0; + this.lastSuccessAtMs = Date.now(); + return; + } + + // Failure: schedule the next retry with capped exponential backoff. + this.consecutiveFailures += 1; + const backoffMs = Math.min( + this.retryBaseMs * 2 ** (this.consecutiveFailures - 1), + this.syncFrequencyMs, + ); + this.nextRetryAtMs = Date.now() + backoffMs; + this.logger.warn( + `OpenChoreo catalog sync failed (attempt ${this.consecutiveFailures}); retrying in ${Math.round( + backoffMs / 1000, + )}s`, + ); + } + /** * Handles an incoming OpenChoreo event and applies a delta mutation. * For `deleted` events, the entity is removed from the catalog. @@ -380,10 +456,11 @@ export class OpenChoreoEntityProvider implements EntityProvider { throw new Error('Connection not initialized'); } - return this.runNew(); + await this.runNew(); } - private async runNew(): Promise { + /** Runs a full sync. Returns true if the mutation was applied, false if the run aborted. */ + private async runNew(): Promise { try { this.logger.info( 'Fetching namespaces and projects from OpenChoreo API (new API)', @@ -1908,8 +1985,10 @@ export class OpenChoreoEntityProvider implements EntityProvider { }); this.logEntityCounts(allEntities, domainEntities.length); + return true; } catch (error) { this.logger.error(`Failed to run OpenChoreoEntityProvider: ${error}`); + return false; } } diff --git a/plugins/catalog-backend-module-openchoreo/src/utils/openChoreoApiClient.ts b/plugins/catalog-backend-module-openchoreo/src/utils/openChoreoApiClient.ts index 2af0bb357..a82fc5897 100644 --- a/plugins/catalog-backend-module-openchoreo/src/utils/openChoreoApiClient.ts +++ b/plugins/catalog-backend-module-openchoreo/src/utils/openChoreoApiClient.ts @@ -6,8 +6,13 @@ export type OpenChoreoApiClient = ReturnType; /** * Builds an authenticated OpenChoreo API client. Acquires a service token - * via the optional `OpenChoreoTokenService`; if token acquisition fails, - * logs a warning and returns an unauthenticated client. + * via the optional `OpenChoreoTokenService`. + * + * When credentials are configured but the token cannot be obtained (e.g. the + * IdP is briefly unreachable at cold start), this throws rather than returning + * an unauthenticated client — every API call would 401 anyway, so failing + * fast lets the caller's retry path re-attempt soon. When no credentials are + * configured (auth disabled), an unauthenticated client is returned as before. */ export async function createAuthenticatedOpenChoreoApiClient(opts: { baseUrl: string; @@ -20,9 +25,8 @@ export async function createAuthenticatedOpenChoreoApiClient(opts: { try { token = await tokenService.getServiceToken(); } catch (error) { - logger.warn( - `Failed to get service token, continuing without auth: ${error}`, - ); + logger.error(`Failed to get OpenChoreo service token: ${error}`); + throw error; } } return createOpenChoreoApiClient({ baseUrl, token, logger }); diff --git a/plugins/openchoreo-common/config.d.ts b/plugins/openchoreo-common/config.d.ts index ca2151fec..cab1df9ed 100644 --- a/plugins/openchoreo-common/config.d.ts +++ b/plugins/openchoreo-common/config.d.ts @@ -129,6 +129,13 @@ export interface Config { */ frequency?: number; + /** + * Seconds between retries while a run is failing. Backs off + * exponentially from this base up to `frequency`. Defaults to 30. + * @visibility backend + */ + retryInterval?: number; + /** * Timeout in seconds for provider operations * @visibility backend From 1567cbd7fd3d8e22773b4fe50269f9c9dbb358a6 Mon Sep 17 00:00:00 2001 From: Stefinie Fernando Date: Fri, 24 Jul 2026 15:28:33 +0530 Subject: [PATCH 2/2] fix: improve logging format for OpenChoreo catalog sync retry attempts Signed-off-by: Stefinie Fernando --- .../src/provider/OpenChoreoEntityProvider.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-backend-module-openchoreo/src/provider/OpenChoreoEntityProvider.ts b/plugins/catalog-backend-module-openchoreo/src/provider/OpenChoreoEntityProvider.ts index b6b7de6a7..11a0dfb56 100644 --- a/plugins/catalog-backend-module-openchoreo/src/provider/OpenChoreoEntityProvider.ts +++ b/plugins/catalog-backend-module-openchoreo/src/provider/OpenChoreoEntityProvider.ts @@ -378,9 +378,9 @@ export class OpenChoreoEntityProvider implements EntityProvider { ); this.nextRetryAtMs = Date.now() + backoffMs; this.logger.warn( - `OpenChoreo catalog sync failed (attempt ${this.consecutiveFailures}); retrying in ${Math.round( - backoffMs / 1000, - )}s`, + `OpenChoreo catalog sync failed (attempt ${ + this.consecutiveFailures + }); retrying in ${Math.round(backoffMs / 1000)}s`, ); }