diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.ts index bda1af70..b0649cd0 100644 --- a/apps/api/src/controllers/v0/crawl-status.ts +++ b/apps/api/src/controllers/v0/crawl-status.ts @@ -56,6 +56,19 @@ export async function crawlStatusController(req: Request, res: Response) { const data = jobs.map(x => Array.isArray(x.returnvalue) ? x.returnvalue[0] : x.returnvalue); + if ( + jobs.length > 0 && + jobs[0].data && + jobs[0].data.pageOptions && + !jobs[0].data.pageOptions.includeRawHtml + ) { + data.forEach(item => { + if (item) { + delete item.rawHtml; + } + }); + } + res.json({ status: jobStatus, current: jobStatuses.filter(x => x === "completed" || x === "failed").length, diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts index 1fe2fd9a..7b94f07a 100644 --- a/apps/api/src/controllers/v1/crawl-status.ts +++ b/apps/api/src/controllers/v1/crawl-status.ts @@ -100,6 +100,14 @@ export async function crawlStatusController(req: RequestWithAuth 0) { + if (!doneJobs[0].data.pageOptions.includeRawHtml) { + for (let ii = 0; ii < doneJobs.length; ii++) { + delete data[ii].rawHtml; + } + } + } + res.status(200).json({ status, completed: doneJobsLength,