From f2ecf0cc36467f97ab0a65dd2987d3f90210ae67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Tue, 12 Nov 2024 19:46:00 +0100 Subject: [PATCH] fix(v0): crawl timeout errors --- apps/api/src/controllers/v0/crawl-status.ts | 2 +- apps/api/src/controllers/v0/crawl.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.ts index 7b6e610a..9c799eeb 100644 --- a/apps/api/src/controllers/v0/crawl-status.ts +++ b/apps/api/src/controllers/v0/crawl-status.ts @@ -75,7 +75,7 @@ export async function crawlStatusController(req: Request, res: Response) { const jobStatus = sc.cancelled ? "failed" : jobStatuses.every(x => x === "completed") ? "completed" : "active"; - const data = jobs.filter(x => x.failedReason !== "Concurreny limit hit").map(x => Array.isArray(x.returnvalue) ? x.returnvalue[0] : x.returnvalue); + const data = jobs.filter(x => x.failedReason !== "Concurreny limit hit" && x.returnvalue !== null).map(x => Array.isArray(x.returnvalue) ? x.returnvalue[0] : x.returnvalue); if ( jobs.length > 0 && diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts index cb7a3ccc..fa7627da 100644 --- a/apps/api/src/controllers/v0/crawl.ts +++ b/apps/api/src/controllers/v0/crawl.ts @@ -138,6 +138,8 @@ export async function crawlController(req: Request, res: Response) { const { scrapeOptions, internalOptions } = fromLegacyScrapeOptions(pageOptions, undefined, undefined); + delete (scrapeOptions as any).timeout; + const sc: StoredCrawl = { originUrl: url, crawlerOptions,