feat: retry catalog sync with exponential backoff to speed up cold-start recovery

.changeset/catalog-sync-retry-backoff.md

@@ -0,0 +1,11 @@
+---
+'@openchoreo/backstage-plugin-catalog-backend-module': patch
+'@openchoreo/openchoreo-auth': patch
+'@openchoreo/backstage-plugin-common': patch
+---
+
+Recover the OpenChoreo catalog sync quickly after a failed run instead of waiting a full sync interval. Previously a failure (e.g. the IdP/API being briefly unreachable during a cluster cold start) aborted the periodic full sync, leaving the catalog empty until the next scheduled run (default 5 minutes) later.
+
+The entity provider now retries a failed sync with capped exponential backoff — starting at `openchoreo.schedule.retryInterval` (new, default 30s) and doubling on each consecutive failure up to `schedule.frequency` — so it recovers within ~30s of the backend returning, backs off gracefully during a sustained outage instead of hammering, and never stops reconciling (the catalog self-heals whenever the backend comes back).
+
+Client-credentials token acquisition also retries transient failures (network errors and 5xx, but not 4xx) with exponential backoff, and the catalog API client now fails fast when a service token cannot be obtained rather than silently issuing unauthenticated requests that are guaranteed to 401.

app-config.production.yaml

@@ -202,6 +202,10 @@ openchoreo:
 
   schedule:
     frequency: ${OPENCHOREO_CATALOG_SYNC_FREQUENCY}
+    # Seconds between retries while a sync is failing; backs off exponentially
+    # up to `frequency`. Environment variable: OPENCHOREO_CATALOG_SYNC_RETRY_INTERVAL
+    # (seconds). When unset it resolves to undefined and the code default (30) applies.
+    retryInterval: ${OPENCHOREO_CATALOG_SYNC_RETRY_INTERVAL}
     timeout: 120 # seconds for timeout (default: 120)
 
   # Whether to wire the OpenChoreo event-driven catalog sync. Mirrors

app-config.yaml

@@ -185,6 +185,7 @@ openchoreo:
 
   schedule:
     frequency: 300 # seconds between runs (default: 300, the periodic full sync acts as a safety net behind the event-forwarder's real-time deltas)
+    retryInterval: 30 # seconds between retries while a sync is failing (default: 30); recovers fast from cold-start dependency outages without waiting a full frequency
     timeout: 120 # seconds for timeout (default: 120)
 
   # Observability tuning (optional).

packages/openchoreo-auth/src/ClientCredentialsProvider.test.ts

@@ -0,0 +1,103 @@
+import { ClientCredentialsProvider } from './ClientCredentialsProvider';
+import { OpenChoreoAuthConfig } from './types';
+
+const authConfig: OpenChoreoAuthConfig = {
+  clientId: 'test-client',
+  clientSecret: 'test-secret',
+  tokenUrl: 'http://idp.test/oauth2/token',
+  scopes: [],
+};
+
+function mkLogger() {
+  return {
+    info: jest.fn(),
+    debug: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn(),
+    child: jest.fn().mockReturnThis(),
+  } as any;
+}
+
+// `access_token` is intentionally not a real JWT; decodeJwt throws and the
+// provider falls back to `expires_in`, which is all these tests need.
+const okResponse = () => ({
+  ok: true,
+  status: 200,
+  statusText: 'OK',
+  json: async () => ({ access_token: 'access-token', expires_in: 3600 }),
+  text: async () => '',
+});
+
+const errorResponse = (status: number) => ({
+  ok: false,
+  status,
+  statusText: 'Error',
+  json: async () => ({}),
+  text: async () => 'error body',
+});
+
+describe('ClientCredentialsProvider', () => {
+  const realFetch = global.fetch;
+
+  afterEach(() => {
+    global.fetch = realFetch;
+    jest.useRealTimers();
+    jest.clearAllMocks();
+  });
+
+  function mockFetch(...outcomes: Array<object | Error>) {
+    const fn = jest.fn();
+    for (const outcome of outcomes) {
+      if (outcome instanceof Error) {
+        fn.mockRejectedValueOnce(outcome);
+      } else {
+        fn.mockResolvedValueOnce(outcome);
+      }
+    }
+    global.fetch = fn as any;
+    return fn;
+  }
+
+  it('returns the token on first success without retrying', async () => {
+    const fetchMock = mockFetch(okResponse());
+    const provider = new ClientCredentialsProvider(authConfig, mkLogger());
+
+    await expect(provider.getToken()).resolves.toBe('access-token');
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+  });
+
+  it('retries a network error and then succeeds', async () => {
+    jest.useFakeTimers();
+    const fetchMock = mockFetch(new Error('fetch failed'), okResponse());
+    const provider = new ClientCredentialsProvider(authConfig, mkLogger());
+
+    const tokenPromise = provider.getToken();
+    await jest.advanceTimersByTimeAsync(1000); // first backoff
+    await expect(tokenPromise).resolves.toBe('access-token');
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+  });
+
+  it('does not retry a 4xx response', async () => {
+    const fetchMock = mockFetch(errorResponse(401));
+    const provider = new ClientCredentialsProvider(authConfig, mkLogger());
+
+    await expect(provider.getToken()).rejects.toThrow(/401/);
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+  });
+
+  it('retries a 5xx response up to the max attempts, then throws', async () => {
+    jest.useFakeTimers();
+    const fetchMock = mockFetch(
+      errorResponse(503),
+      errorResponse(503),
+      errorResponse(503),
+    );
+    const provider = new ClientCredentialsProvider(authConfig, mkLogger());
+
+    const tokenPromise = provider.getToken();
+    tokenPromise.catch(() => {}); // avoid unhandled rejection while timers advance
+    await jest.advanceTimersByTimeAsync(1000 + 2000); // both backoffs
+    await expect(tokenPromise).rejects.toThrow(/503/);
+    expect(fetchMock).toHaveBeenCalledTimes(3);
+  });
+});

packages/openchoreo-auth/src/ClientCredentialsProvider.ts

@@ -8,6 +8,25 @@ import { OpenChoreoAuthConfig, CachedToken, TokenResponse } from './types';
  */
 const TOKEN_EXPIRY_BUFFER_MS = 60 * 1000;
 
+/** Number of token-fetch attempts before giving up. */
+const TOKEN_FETCH_MAX_ATTEMPTS = 3;
+/** Base backoff between token-fetch retries (doubles each attempt). */
+const TOKEN_FETCH_BASE_DELAY_MS = 1000;
+
+const delay = (ms: number): Promise<void> =>
+  new Promise(resolve => setTimeout(resolve, ms));
+
+/**
+ * Error from a single token request, tagged with whether retrying could help.
+ * Network failures and 5xx are retryable; 4xx (e.g. bad credentials) are not.
+ */
+class TokenRequestError extends Error {
+  constructor(message: string, readonly retryable: boolean) {
+    super(message);
+    this.name = 'TokenRequestError';
+  }
+}
+
 /**
  * Provides OAuth2 access tokens using the client credentials grant flow.
  * Used for background tasks that don't have a user context.
@@ -61,9 +80,41 @@ export class ClientCredentialsProvider {
   }
 
   /**
-   * Fetches a new access token from the OAuth2 token endpoint.
+   * Fetches a new access token, retrying transient failures with exponential
+   * backoff. Only network errors and 5xx responses are retried; 4xx (e.g. bad
+   * credentials) fail immediately since retrying cannot help.
    */
   private async fetchNewToken(): Promise<string> {
+    let lastError: unknown;
+    for (let attempt = 1; attempt <= TOKEN_FETCH_MAX_ATTEMPTS; attempt++) {
+      try {
+        return await this.requestToken();
+      } catch (error) {
+        lastError = error;
+        const retryable =
+          !(error instanceof TokenRequestError) || error.retryable;
+        if (!retryable || attempt === TOKEN_FETCH_MAX_ATTEMPTS) {
+          break;
+        }
+        const backoffMs = TOKEN_FETCH_BASE_DELAY_MS * 2 ** (attempt - 1);
+        this.logger.warn(
+          `Client credentials token fetch attempt ${attempt}/${TOKEN_FETCH_MAX_ATTEMPTS} failed; retrying in ${backoffMs}ms: ${error}`,
+        );
+        await delay(backoffMs);
+      }
+    }
+    this.logger.error(
+      'Failed to fetch client credentials token',
+      lastError as Error,
+    );
+    throw lastError;
+  }
+
+  /**
+   * Performs a single token request against the OAuth2 token endpoint.
+   * Throws a {@link TokenRequestError} on non-ok responses.
+   */
+  private async requestToken(): Promise<string> {
     this.logger.debug('Fetching new client credentials token');
 
     const { clientId, clientSecret, tokenUrl, scopes } = this.config;
@@ -78,74 +129,67 @@ export class ClientCredentialsProvider {
       params.set('scope', scopes.join(' '));
     }
 
-    try {
-      const response = await fetch(tokenUrl, {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/x-www-form-urlencoded',
-        },
-        body: params.toString(),
-      });
-
-      if (!response.ok) {
-        const errorText = await response.text();
-        throw new Error(
-          `Token request failed: ${response.status} ${response.statusText} - ${errorText}`,
-        );
-      }
+    const response = await fetch(tokenUrl, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/x-www-form-urlencoded',
+      },
+      body: params.toString(),
+    });
 
-      const tokenResponse: TokenResponse = await response.json();
+    if (!response.ok) {
+      const errorText = await response.text();
+      throw new TokenRequestError(
+        `Token request failed: ${response.status} ${response.statusText} - ${errorText}`,
+        response.status >= 500,
+      );
+    }
 
-      // Decode JWT to extract the actual expiration time from the exp claim
-      // This is more accurate than using expires_in, which doesn't account for
-      // time elapsed between token issuance and response receipt
-      let expiresAt: number;
-      try {
-        const decodedToken = decodeJwt(tokenResponse.access_token);
-        if (decodedToken.exp) {
-          // exp claim is in seconds, convert to milliseconds
-          expiresAt = decodedToken.exp * 1000;
-          this.logger.debug(
-            `Using JWT exp claim for expiry: ${expiresAt} (${new Date(
-              expiresAt,
-            ).toISOString()})`,
-          );
-        } else {
-          // Fallback to expires_in if exp claim is missing
-          expiresAt = Date.now() + tokenResponse.expires_in * 1000;
-          this.logger.debug(
-            `JWT exp claim missing, falling back to expires_in calculation: ${expiresAt}`,
-          );
-        }
-      } catch (error) {
-        // Fallback to expires_in if JWT decoding fails
+    const tokenResponse: TokenResponse = await response.json();
+
+    // Decode JWT to extract the actual expiration time from the exp claim
+    // This is more accurate than using expires_in, which doesn't account for
+    // time elapsed between token issuance and response receipt
+    let expiresAt: number;
+    try {
+      const decodedToken = decodeJwt(tokenResponse.access_token);
+      if (decodedToken.exp) {
+        // exp claim is in seconds, convert to milliseconds
+        expiresAt = decodedToken.exp * 1000;
+        this.logger.debug(
+          `Using JWT exp claim for expiry: ${expiresAt} (${new Date(
+            expiresAt,
+          ).toISOString()})`,
+        );
+      } else {
+        // Fallback to expires_in if exp claim is missing
         expiresAt = Date.now() + tokenResponse.expires_in * 1000;
         this.logger.debug(
-          `Failed to decode JWT token, falling back to expires_in calculation: ${error}`,
+          `JWT exp claim missing, falling back to expires_in calculation: ${expiresAt}`,
         );
       }
-
-      // Cache the token
-      this.cachedToken = {
-        accessToken: tokenResponse.access_token,
-        expiresAt,
-      };
-
-      const expiresInSeconds = Math.round((expiresAt - Date.now()) / 1000);
-      this.logger.debug(
-        `Successfully obtained client credentials token, expires in ${expiresInSeconds}s at ${new Date(
-          expiresAt,
-        ).toISOString()}`,
-      );
-
-      return tokenResponse.access_token;
     } catch (error) {
-      this.logger.error(
-        'Failed to fetch client credentials token',
-        error as Error,
+      // Fallback to expires_in if JWT decoding fails
+      expiresAt = Date.now() + tokenResponse.expires_in * 1000;
+      this.logger.debug(
+        `Failed to decode JWT token, falling back to expires_in calculation: ${error}`,
       );
-      throw error;
     }
+
+    // Cache the token
+    this.cachedToken = {
+      accessToken: tokenResponse.access_token,
+      expiresAt,
+    };
+
+    const expiresInSeconds = Math.round((expiresAt - Date.now()) / 1000);
+    this.logger.debug(
+      `Successfully obtained client credentials token, expires in ${expiresInSeconds}s at ${new Date(
+        expiresAt,
+      ).toISOString()}`,
+    );
+
+    return tokenResponse.access_token;
   }
 
   /**

plugins/catalog-backend-module-openchoreo/src/module.ts

@@ -96,8 +96,13 @@ export const catalogModuleOpenchoreo = createBackendModule({
         urlReader,
       }) {
         const openchoreoConfig = config.getOptionalConfig('openchoreo');
-        const frequency =
-          openchoreoConfig?.getOptionalNumber('schedule.frequency') ?? 300;
+        // The scheduled task fires at this fast retry cadence; the provider
+        // throttles so a healthy sync still only runs every
+        // `schedule.frequency` seconds (which the provider reads itself),
+        // but a failed run (e.g. dependency unreachable at cold start)
+        // retries within `retryInterval` instead of waiting a full frequency.
+        const retryInterval =
+          openchoreoConfig?.getOptionalNumber('schedule.retryInterval') ?? 30;
         const timeout =
           openchoreoConfig?.getOptionalNumber('schedule.timeout') ?? 120;
         // Whether to wire up the event-driven sync. Defaults to true so
@@ -111,7 +116,7 @@ export const catalogModuleOpenchoreo = createBackendModule({
           openchoreoConfig?.getOptionalBoolean('events.enabled') ?? true;
 
         const taskRunner = scheduler.createScheduledTaskRunner({
-          frequency: { seconds: frequency },
+          frequency: { seconds: retryInterval },
           timeout: { seconds: timeout },
         });