diff --git a/apps/api/src/lib/validateUrl.ts b/apps/api/src/lib/validateUrl.ts index a73c06f0db..3dcdd7119b 100644 --- a/apps/api/src/lib/validateUrl.ts +++ b/apps/api/src/lib/validateUrl.ts @@ -200,6 +200,7 @@ export async function resolveRedirects( const targetUrl = protocolIncluded(url) ? url : `http://${url}`; const methods = ["HEAD", "GET"] as const; + const dispatcher = getSecureDispatcher(false); for (const method of methods) { const signal = abort @@ -210,7 +211,7 @@ export async function resolveRedirects( const response = await undici.fetch(targetUrl, { method, redirect: "follow", - dispatcher: getSecureDispatcher(false), + dispatcher, signal, }); diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts index 3b0ad57e5c..4ef7ca8ad3 100644 --- a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts +++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts @@ -60,11 +60,8 @@ function attachSecurityCheck(agent: undici.Dispatcher) { }); } -// Dispatcher WITH cookie handling (for scraping - needs cookies for auth flows) function makeSecureDispatcher(skipTlsVerification: boolean) { - const baseAgent = createBaseAgent(skipTlsVerification); - const cookieJar = new CookieJar(); - const agent = baseAgent.compose(cookie({ jar: cookieJar })); + const agent = createBaseAgent(skipTlsVerification); attachSecurityCheck(agent); return agent; } @@ -82,8 +79,13 @@ const secureDispatcherNoCookies = makeSecureDispatcherNoCookies(false); const secureDispatcherNoCookiesSkipTlsVerification = makeSecureDispatcherNoCookies(true); -export const getSecureDispatcher = (skipTlsVerification: boolean = false) => - skipTlsVerification ? secureDispatcherSkipTlsVerification : secureDispatcher; +export const getSecureDispatcher = (skipTlsVerification: boolean = false) => { + const dispatcher = skipTlsVerification + ? secureDispatcherSkipTlsVerification + : secureDispatcher; + + return dispatcher.compose(cookie({ jar: new CookieJar() })); +}; // Use this for webhook delivery to avoid sending empty cookie headers export const getSecureDispatcherNoCookies = ( diff --git a/apps/api/src/search/v2/ddgsearch.ts b/apps/api/src/search/v2/ddgsearch.ts index 9838423acb..9b2571f4f7 100644 --- a/apps/api/src/search/v2/ddgsearch.ts +++ b/apps/api/src/search/v2/ddgsearch.ts @@ -130,6 +130,7 @@ export async function ddgSearch( const results: WebSearchResult[] = []; const seenUrls = new Set(); + const dispatcher = getSecureDispatcher(false); let isFirstPage = true; let nextPageData: URLSearchParams | null = params; @@ -149,7 +150,7 @@ export async function ddgSearch( response = await undici.fetch( `https://html.duckduckgo.com/html?${params.toString()}`, { - dispatcher: getSecureDispatcher(false), + dispatcher, redirect: "follow", headers: { "User-Agent": userAgent, @@ -166,7 +167,7 @@ export async function ddgSearch( response = await undici.fetch(`https://html.duckduckgo.com/html`, { method: "POST", body: nextPageData.toString(), - dispatcher: getSecureDispatcher(false), + dispatcher, redirect: "follow", headers: { "User-Agent": userAgent,