Skip to content

Commit 82b874c

Browse files
waleedlatif1waleedlatif1
committed
improvements
1 parent 3eaf5ba commit 82b874c

File tree

10 files changed

+158
-82
lines changed

10 files changed

+158
-82
lines changed

‎.claude/commands/add-connector.md‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ connectors/{service}/
2828
```typescript
2929
import { createLogger } from '@sim/logger'
3030
import { {Service}Icon } from '@/components/icons'
31+
import { fetchWithRetry } from '@/lib/knowledge/documents/utils'
3132
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
3233

3334
const logger = createLogger('{Service}Connector')
@@ -179,6 +180,25 @@ mapTags: (metadata: Record<string, unknown>): Record<string, unknown> => {
179180
}
180181
```
181182

183+
## External API Calls — Use `fetchWithRetry`
184+
185+
All external API calls must use `fetchWithRetry` from `@/lib/knowledge/documents/utils` instead of raw `fetch()`. This provides exponential backoff with retries on 429/502/503/504 errors. It returns a standard `Response` — all `.ok`, `.json()`, `.text()` checks work unchanged.
186+
187+
For `validateConfig` (user-facing, called on save), pass `VALIDATE_RETRY_OPTIONS` to cap wait time at ~7s. Background operations (`listDocuments`, `getDocument`) use the built-in defaults (5 retries, ~31s max).
188+
189+
```typescript
190+
import { VALIDATE_RETRY_OPTIONS, fetchWithRetry } from '@/lib/knowledge/documents/utils'
191+
192+
// Background sync — use defaults
193+
const response = await fetchWithRetry(url, {
194+
method: 'GET',
195+
headers: { Authorization: `Bearer ${accessToken}` },
196+
})
197+
198+
// validateConfig — tighter retry budget
199+
const response = await fetchWithRetry(url, { ... }, VALIDATE_RETRY_OPTIONS)
200+
```
201+
182202
## sourceUrl
183203

184204
If `ExternalDocument.sourceUrl` is set, the sync engine stores it on the document record. Always construct the full URL (not a relative path).
@@ -235,6 +255,7 @@ See `apps/sim/connectors/confluence/confluence.ts` for a complete example with:
235255
- [ ] `tagDefinitions` declared for each semantic key returned by `mapTags`
236256
- [ ] `mapTags` implemented if source has useful metadata (labels, dates, versions)
237257
- [ ] `validateConfig` verifies the source is accessible
258+
- [ ] All external API calls use `fetchWithRetry` (not raw `fetch`)
238259
- [ ] All optional config fields validated in `validateConfig`
239260
- [ ] Icon exists in `components/icons.tsx` (or asked user to provide SVG)
240261
- [ ] Registered in `connectors/registry.ts`

‎apps/sim/connectors/airtable/airtable.ts‎

Lines changed: 22 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { createLogger } from '@sim/logger'
22
import { AirtableIcon } from '@/components/icons'
3+
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
34
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
45

56
const logger = createLogger('AirtableConnector')
@@ -162,7 +163,7 @@ export const airtableConnector: ConnectorConfig = {
162163
view: viewId ?? 'default',
163164
})
164165

165-
const response = await fetch(url, {
166+
const response = await fetchWithRetry(url, {
166167
method: 'GET',
167168
headers: {
168169
Authorization: `Bearer ${accessToken}`,
@@ -211,7 +212,7 @@ export const airtableConnector: ConnectorConfig = {
211212
const encodedTable = encodeURIComponent(tableIdOrName)
212213
const url = `${AIRTABLE_API}/${baseId}/${encodedTable}/${externalId}`
213214

214-
const response = await fetch(url, {
215+
const response = await fetchWithRetry(url, {
215216
method: 'GET',
216217
headers: {
217218
Authorization: `Bearer ${accessToken}`,
@@ -251,12 +252,16 @@ export const airtableConnector: ConnectorConfig = {
251252
// Verify base and table are accessible by fetching 1 record
252253
const encodedTable = encodeURIComponent(tableIdOrName)
253254
const url = `${AIRTABLE_API}/${baseId}/${encodedTable}?pageSize=1`
254-
const response = await fetch(url, {
255-
method: 'GET',
256-
headers: {
257-
Authorization: `Bearer ${accessToken}`,
255+
const response = await fetchWithRetry(
256+
url,
257+
{
258+
method: 'GET',
259+
headers: {
260+
Authorization: `Bearer ${accessToken}`,
261+
},
258262
},
259-
})
263+
VALIDATE_RETRY_OPTIONS
264+
)
260265

261266
if (!response.ok) {
262267
const errorText = await response.text()
@@ -273,12 +278,16 @@ export const airtableConnector: ConnectorConfig = {
273278
const viewId = sourceConfig.viewId as string | undefined
274279
if (viewId) {
275280
const viewUrl = `${AIRTABLE_API}/${baseId}/${encodedTable}?pageSize=1&view=${encodeURIComponent(viewId)}`
276-
const viewResponse = await fetch(viewUrl, {
277-
method: 'GET',
278-
headers: {
279-
Authorization: `Bearer ${accessToken}`,
281+
const viewResponse = await fetchWithRetry(
282+
viewUrl,
283+
{
284+
method: 'GET',
285+
headers: {
286+
Authorization: `Bearer ${accessToken}`,
287+
},
280288
},
281-
})
289+
VALIDATE_RETRY_OPTIONS
290+
)
282291
if (!viewResponse.ok) {
283292
return { valid: false, error: `View "${viewId}" not found in table "${tableIdOrName}"` }
284293
}
@@ -354,7 +363,7 @@ async function fetchFieldNames(
354363

355364
try {
356365
const url = `${AIRTABLE_API}/meta/bases/${baseId}/tables`
357-
const response = await fetch(url, {
366+
const response = await fetchWithRetry(url, {
358367
method: 'GET',
359368
headers: {
360369
Authorization: `Bearer ${accessToken}`,

‎apps/sim/connectors/confluence/confluence.ts‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { createLogger } from '@sim/logger'
22
import { ConfluenceIcon } from '@/components/icons'
3+
import { fetchWithRetry } from '@/lib/knowledge/documents/utils'
34
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
45
import { getConfluenceCloudId } from '@/tools/confluence/utils'
56

@@ -45,7 +46,7 @@ async function fetchLabelsForPages(
4546
pageIds.map(async (pageId) => {
4647
try {
4748
const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${pageId}/labels`
48-
const response = await fetch(url, {
49+
const response = await fetchWithRetry(url, {
4950
method: 'GET',
5051
headers: {
5152
Accept: 'application/json',
@@ -226,7 +227,7 @@ export const confluenceConnector: ConnectorConfig = {
226227

227228
const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/pages/${externalId}?body-format=storage`
228229

229-
const response = await fetch(url, {
230+
const response = await fetchWithRetry(url, {
230231
method: 'GET',
231232
headers: {
232233
Accept: 'application/json',
@@ -342,7 +343,7 @@ async function listDocumentsV2(
342343

343344
logger.info(`Listing ${endpoint} in space ${spaceKey} (ID: ${spaceId})`)
344345

345-
const response = await fetch(url, {
346+
const response = await fetchWithRetry(url, {
346347
method: 'GET',
347348
headers: {
348349
Accept: 'application/json',
@@ -543,7 +544,7 @@ async function listDocumentsViaCql(
543544

544545
logger.info(`Searching Confluence via CQL: ${cql}`, { start, limit })
545546

546-
const response = await fetch(url, {
547+
const response = await fetchWithRetry(url, {
547548
method: 'GET',
548549
headers: {
549550
Accept: 'application/json',
@@ -588,7 +589,7 @@ async function resolveSpaceId(
588589
): Promise<string> {
589590
const url = `https://api.atlassian.com/ex/confluence/${cloudId}/wiki/api/v2/spaces?keys=${encodeURIComponent(spaceKey)}&limit=1`
590591

591-
const response = await fetch(url, {
592+
const response = await fetchWithRetry(url, {
592593
method: 'GET',
593594
headers: {
594595
Accept: 'application/json',

‎apps/sim/connectors/github/github.ts‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { createLogger } from '@sim/logger'
22
import { GithubIcon } from '@/components/icons'
3+
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
34
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
45

56
const logger = createLogger('GitHubConnector')
@@ -73,7 +74,7 @@ async function fetchTree(
7374
): Promise<TreeItem[]> {
7475
const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/git/trees/${encodeURIComponent(branch)}?recursive=1`
7576

76-
const response = await fetch(url, {
77+
const response = await fetchWithRetry(url, {
7778
method: 'GET',
7879
headers: {
7980
Accept: 'application/vnd.github+json',
@@ -108,7 +109,7 @@ async function fetchBlobContent(
108109
): Promise<string> {
109110
const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/git/blobs/${sha}`
110111

111-
const response = await fetch(url, {
112+
const response = await fetchWithRetry(url, {
112113
method: 'GET',
113114
headers: {
114115
Accept: 'application/vnd.github+json',
@@ -282,7 +283,7 @@ export const githubConnector: ConnectorConfig = {
282283

283284
try {
284285
const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/contents/${encodeURIComponent(path)}?ref=${encodeURIComponent(branch)}`
285-
const response = await fetch(url, {
286+
const response = await fetchWithRetry(url, {
286287
method: 'GET',
287288
headers: {
288289
Accept: 'application/vnd.github+json',
@@ -358,14 +359,18 @@ export const githubConnector: ConnectorConfig = {
358359
try {
359360
// Verify repo and branch are accessible
360361
const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/branches/${encodeURIComponent(branch)}`
361-
const response = await fetch(url, {
362-
method: 'GET',
363-
headers: {
364-
Accept: 'application/vnd.github+json',
365-
Authorization: `Bearer ${accessToken}`,
366-
'X-GitHub-Api-Version': '2022-11-28',
362+
const response = await fetchWithRetry(
363+
url,
364+
{
365+
method: 'GET',
366+
headers: {
367+
Accept: 'application/vnd.github+json',
368+
Authorization: `Bearer ${accessToken}`,
369+
'X-GitHub-Api-Version': '2022-11-28',
370+
},
367371
},
368-
})
372+
VALIDATE_RETRY_OPTIONS
373+
)
369374

370375
if (response.status === 404) {
371376
return {

‎apps/sim/connectors/google-drive/google-drive.ts‎

Lines changed: 25 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { createLogger } from '@sim/logger'
22
import { GoogleDriveIcon } from '@/components/icons'
3+
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
34
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
45

56
const logger = createLogger('GoogleDriveConnector')
@@ -61,7 +62,7 @@ async function exportGoogleWorkspaceFile(
6162

6263
const url = `https://www.googleapis.com/drive/v3/files/${fileId}/export?mimeType=${encodeURIComponent(exportMimeType)}`
6364

64-
const response = await fetch(url, {
65+
const response = await fetchWithRetry(url, {
6566
method: 'GET',
6667
headers: { Authorization: `Bearer ${accessToken}` },
6768
})
@@ -76,7 +77,7 @@ async function exportGoogleWorkspaceFile(
7677
async function downloadTextFile(accessToken: string, fileId: string): Promise<string> {
7778
const url = `https://www.googleapis.com/drive/v3/files/${fileId}?alt=media`
7879

79-
const response = await fetch(url, {
80+
const response = await fetchWithRetry(url, {
8081
method: 'GET',
8182
headers: { Authorization: `Bearer ${accessToken}` },
8283
})
@@ -266,7 +267,7 @@ export const googleDriveConnector: ConnectorConfig = {
266267

267268
logger.info('Listing Google Drive files', { query, cursor: cursor ?? 'initial' })
268269

269-
const response = await fetch(url, {
270+
const response = await fetchWithRetry(url, {
270271
method: 'GET',
271272
headers: {
272273
Authorization: `Bearer ${accessToken}`,
@@ -310,7 +311,7 @@ export const googleDriveConnector: ConnectorConfig = {
310311
'id,name,mimeType,modifiedTime,createdTime,webViewLink,parents,owners,size,starred,trashed'
311312
const url = `https://www.googleapis.com/drive/v3/files/${externalId}?fields=${encodeURIComponent(fields)}&supportsAllDrives=true`
312313

313-
const response = await fetch(url, {
314+
const response = await fetchWithRetry(url, {
314315
method: 'GET',
315316
headers: {
316317
Authorization: `Bearer ${accessToken}`,
@@ -346,13 +347,17 @@ export const googleDriveConnector: ConnectorConfig = {
346347
if (folderId?.trim()) {
347348
// Verify the folder exists and is accessible
348349
const url = `https://www.googleapis.com/drive/v3/files/${folderId.trim()}?fields=id,name,mimeType&supportsAllDrives=true`
349-
const response = await fetch(url, {
350-
method: 'GET',
351-
headers: {
352-
Authorization: `Bearer ${accessToken}`,
353-
Accept: 'application/json',
350+
const response = await fetchWithRetry(
351+
url,
352+
{
353+
method: 'GET',
354+
headers: {
355+
Authorization: `Bearer ${accessToken}`,
356+
Accept: 'application/json',
357+
},
354358
},
355-
})
359+
VALIDATE_RETRY_OPTIONS
360+
)
356361

357362
if (!response.ok) {
358363
if (response.status === 404) {
@@ -368,13 +373,17 @@ export const googleDriveConnector: ConnectorConfig = {
368373
} else {
369374
// Verify basic Drive access by listing one file
370375
const url = 'https://www.googleapis.com/drive/v3/files?pageSize=1&fields=files(id)'
371-
const response = await fetch(url, {
372-
method: 'GET',
373-
headers: {
374-
Authorization: `Bearer ${accessToken}`,
375-
Accept: 'application/json',
376+
const response = await fetchWithRetry(
377+
url,
378+
{
379+
method: 'GET',
380+
headers: {
381+
Authorization: `Bearer ${accessToken}`,
382+
Accept: 'application/json',
383+
},
376384
},
377-
})
385+
VALIDATE_RETRY_OPTIONS
386+
)
378387

379388
if (!response.ok) {
380389
return { valid: false, error: `Failed to access Google Drive: ${response.status}` }

‎apps/sim/connectors/jira/jira.ts‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { createLogger } from '@sim/logger'
22
import { JiraIcon } from '@/components/icons'
3+
import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
34
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
45
import { extractAdfText, getJiraCloudId } from '@/tools/jira/utils'
56

@@ -159,7 +160,7 @@ export const jiraConnector: ConnectorConfig = {
159160

160161
logger.info(`Listing Jira issues for project ${projectKey}`, { startAt })
161162

162-
const response = await fetch(url, {
163+
const response = await fetchWithRetry(url, {
163164
method: 'GET',
164165
headers: {
165166
Accept: 'application/json',
@@ -210,7 +211,7 @@ export const jiraConnector: ConnectorConfig = {
210211

211212
const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${externalId}?${params.toString()}`
212213

213-
const response = await fetch(url, {
214+
const response = await fetchWithRetry(url, {
214215
method: 'GET',
215216
headers: {
216217
Accept: 'application/json',
@@ -259,13 +260,17 @@ export const jiraConnector: ConnectorConfig = {
259260
params.append('maxResults', '0')
260261

261262
const url = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/search?${params.toString()}`
262-
const response = await fetch(url, {
263-
method: 'GET',
264-
headers: {
265-
Accept: 'application/json',
266-
Authorization: `Bearer ${accessToken}`,
263+
const response = await fetchWithRetry(
264+
url,
265+
{
266+
method: 'GET',
267+
headers: {
268+
Accept: 'application/json',
269+
Authorization: `Bearer ${accessToken}`,
270+
},
267271
},
268-
})
272+
VALIDATE_RETRY_OPTIONS
273+
)
269274

270275
if (!response.ok) {
271276
const errorText = await response.text()

‎apps/sim/connectors/linear/linear.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { createLogger } from '@sim/logger'
22
import { LinearIcon } from '@/components/icons'
3+
import { fetchWithRetry } from '@/lib/knowledge/documents/utils'
34
import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
45

56
const logger = createLogger('LinearConnector')
@@ -46,7 +47,7 @@ async function linearGraphQL(
4647
query: string,
4748
variables?: Record<string, unknown>
4849
): Promise<Record<string, unknown>> {
49-
const response = await fetch(LINEAR_API, {
50+
const response = await fetchWithRetry(LINEAR_API, {
5051
method: 'POST',
5152
headers: {
5253
'Content-Type': 'application/json',

0 commit comments

Comments
 (0)