Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

fix(scrapeURL/fire-engine): timeout-less scrape support (initial)

Browse Source
This commit is contained in:
Gergő Móricz 2024-11-15 20:15:25 +01:00
parent 350d00d27a
commit 4cddcd5206
2 changed files with 7 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
View File

@ -87,6 +87,7 @@ export async function scrapeURLWithFireEngineChromeCDP(meta: Meta): Promise<Engi
priority: meta.internalOptions.priority, priority: meta.internalOptions.priority,
geolocation: meta.options.geolocation, geolocation: meta.options.geolocation,
mobile: meta.options.mobile, mobile: meta.options.mobile,
timeout: meta.options.timeout === undefined ? 300000 : undefined, // TODO: better timeout logic
// TODO: scrollXPaths // TODO: scrollXPaths
}; };
@ -140,6 +141,8 @@ export async function scrapeURLWithFireEnginePlaywright(meta: Meta): Promise<Eng
fullPageScreenshot: meta.options.formats.includes("screenshot@fullPage"), fullPageScreenshot: meta.options.formats.includes("screenshot@fullPage"),
wait: meta.options.waitFor, wait: meta.options.waitFor,
geolocation: meta.options.geolocation, geolocation: meta.options.geolocation,
timeout: meta.options.timeout === undefined ? 300000 : undefined, // TODO: better timeout logic
}; };
let response = await performFireEngineScrape( let response = await performFireEngineScrape(
@ -179,6 +182,8 @@ export async function scrapeURLWithFireEngineTLSClient(meta: Meta): Promise<Engi
atsv: meta.internalOptions.atsv, atsv: meta.internalOptions.atsv,
geolocation: meta.options.geolocation, geolocation: meta.options.geolocation,
disableJsDom: meta.internalOptions.v0DisableJsDom, disableJsDom: meta.internalOptions.v0DisableJsDom,
timeout: meta.options.timeout === undefined ? 300000 : undefined, // TODO: better timeout logic
}; };
let response = await performFireEngineScrape( let response = await performFireEngineScrape(

2
apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
View File

@ -25,6 +25,8 @@ export type FireEngineScrapeRequestCommon = {
logRequest?: boolean; // default: true logRequest?: boolean; // default: true
instantReturn?: boolean; // default: false instantReturn?: boolean; // default: false
geolocation?: { country?: string; languages?: string[]; }; geolocation?: { country?: string; languages?: string[]; };
timeout?: number;
} }
export type FireEngineScrapeRequestChromeCDP = { export type FireEngineScrapeRequestChromeCDP = {