Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 03:32:22 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

fix(scrapeURL/fire-engine): wait longer if timeout is not specified
Some checks are pending
Deploy Images to GHCR / push-app-image (push) Waiting to run
Details

Browse Source
This commit is contained in:
Gergő Móricz 2024-11-15 20:25:16 +01:00
parent 4cddcd5206
commit 63787bc504
1 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
View File

@ -96,7 +96,9 @@ export async function scrapeURLWithFireEngineChromeCDP(meta: Meta): Promise<Engi
let response = await performFireEngineScrape( let response = await performFireEngineScrape(
meta.logger.child({ method: "scrapeURLWithFireEngineChromeCDP/callFireEngine", request }), meta.logger.child({ method: "scrapeURLWithFireEngineChromeCDP/callFireEngine", request }),
request, request,
defaultTimeout + totalWait, meta.options.timeout !== undefined
? defaultTimeout + totalWait
: Infinity, // TODO: better timeout handling
); );
specialtyScrapeCheck(meta.logger.child({ method: "scrapeURLWithFireEngineChromeCDP/specialtyScrapeCheck" }), response.responseHeaders); specialtyScrapeCheck(meta.logger.child({ method: "scrapeURLWithFireEngineChromeCDP/specialtyScrapeCheck" }), response.responseHeaders);
@ -148,7 +150,9 @@ export async function scrapeURLWithFireEnginePlaywright(meta: Meta): Promise<Eng
let response = await performFireEngineScrape( let response = await performFireEngineScrape(
meta.logger.child({ method: "scrapeURLWithFireEngineChromeCDP/callFireEngine", request }), meta.logger.child({ method: "scrapeURLWithFireEngineChromeCDP/callFireEngine", request }),
request, request,
defaultTimeout + meta.options.waitFor meta.options.timeout !== undefined
? defaultTimeout + meta.options.waitFor
: Infinity, // TODO: better timeout handling
); );
specialtyScrapeCheck(meta.logger.child({ method: "scrapeURLWithFireEnginePlaywright/specialtyScrapeCheck" }), response.responseHeaders); specialtyScrapeCheck(meta.logger.child({ method: "scrapeURLWithFireEnginePlaywright/specialtyScrapeCheck" }), response.responseHeaders);
@ -182,13 +186,16 @@ export async function scrapeURLWithFireEngineTLSClient(meta: Meta): Promise<Engi
atsv: meta.internalOptions.atsv, atsv: meta.internalOptions.atsv,
geolocation: meta.options.geolocation, geolocation: meta.options.geolocation,
disableJsDom: meta.internalOptions.v0DisableJsDom, disableJsDom: meta.internalOptions.v0DisableJsDom,
timeout: meta.options.timeout === undefined ? 300000 : undefined, // TODO: better timeout logic timeout: meta.options.timeout === undefined ? 300000 : undefined, // TODO: better timeout logic
}; };
let response = await performFireEngineScrape( let response = await performFireEngineScrape(
meta.logger.child({ method: "scrapeURLWithFireEngineChromeCDP/callFireEngine", request }), meta.logger.child({ method: "scrapeURLWithFireEngineChromeCDP/callFireEngine", request }),
request, request,
meta.options.timeout !== undefined
? defaultTimeout
: Infinity, // TODO: better timeout handling
); );
specialtyScrapeCheck(meta.logger.child({ method: "scrapeURLWithFireEngineTLSClient/specialtyScrapeCheck" }), response.responseHeaders); specialtyScrapeCheck(meta.logger.child({ method: "scrapeURLWithFireEngineTLSClient/specialtyScrapeCheck" }), response.responseHeaders);