Skip to content

[pull] main from firecrawl:main #365

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pull merged 1 commit into from
Aug 12, 2026
+13 −9
Merged

[pull] main from firecrawl:main #365

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion apps/api/src/lib/validateUrl.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,7 @@ export async function resolveRedirects(
const targetUrl = protocolIncluded(url) ? url : `http://${url}`;

const methods = ["HEAD", "GET"] as const;
const dispatcher = getSecureDispatcher(false);

for (const method of methods) {
const signal = abort
Expand All @@ -210,7 +211,7 @@ export async function resolveRedirects(
const response = await undici.fetch(targetUrl, {
method,
redirect: "follow",
dispatcher: getSecureDispatcher(false),
dispatcher,
signal,
});

Expand Down
14 changes: 8 additions & 6 deletions apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,11 +60,8 @@ function attachSecurityCheck(agent: undici.Dispatcher) {
});
}

// Dispatcher WITH cookie handling (for scraping - needs cookies for auth flows)
function makeSecureDispatcher(skipTlsVerification: boolean) {
const baseAgent = createBaseAgent(skipTlsVerification);
const cookieJar = new CookieJar();
const agent = baseAgent.compose(cookie({ jar: cookieJar }));
const agent = createBaseAgent(skipTlsVerification);
attachSecurityCheck(agent);
return agent;
}
Expand All @@ -82,8 +79,13 @@ const secureDispatcherNoCookies = makeSecureDispatcherNoCookies(false);
const secureDispatcherNoCookiesSkipTlsVerification =
makeSecureDispatcherNoCookies(true);

export const getSecureDispatcher = (skipTlsVerification: boolean = false) =>
skipTlsVerification ? secureDispatcherSkipTlsVerification : secureDispatcher;
export const getSecureDispatcher = (skipTlsVerification: boolean = false) => {
const dispatcher = skipTlsVerification
? secureDispatcherSkipTlsVerification
: secureDispatcher;

return dispatcher.compose(cookie({ jar: new CookieJar() }));
};

// Use this for webhook delivery to avoid sending empty cookie headers
export const getSecureDispatcherNoCookies = (
Expand Down
5 changes: 3 additions & 2 deletions apps/api/src/search/v2/ddgsearch.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,7 @@ export async function ddgSearch(

const results: WebSearchResult[] = [];
const seenUrls = new Set<string>();
const dispatcher = getSecureDispatcher(false);
let isFirstPage = true;
let nextPageData: URLSearchParams | null = params;

Expand All @@ -149,7 +150,7 @@ export async function ddgSearch(
response = await undici.fetch(
`https://html.duckduckgo.com/html?${params.toString()}`,
{
dispatcher: getSecureDispatcher(false),
dispatcher,
redirect: "follow",
headers: {
"User-Agent": userAgent,
Expand All @@ -166,7 +167,7 @@ export async function ddgSearch(
response = await undici.fetch(`https://html.duckduckgo.com/html`, {
method: "POST",
body: nextPageData.toString(),
dispatcher: getSecureDispatcher(false),
dispatcher,
redirect: "follow",
headers: {
"User-Agent": userAgent,
Expand Down