Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 19:58:08 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #582 from mendableai/fix--rawHtml-return-on-crawl
Some checks are pending
Fly Deploy / Pre-deploy checks (push) Waiting to run
Details
Fly Deploy / Test Suite (push) Blocked by required conditions
Details
Fly Deploy / Python SDK Tests (push) Blocked by required conditions
Details
Fly Deploy / JavaScript SDK Tests (push) Blocked by required conditions
Details
Fly Deploy / Go SDK Tests (push) Blocked by required conditions
Details
Fly Deploy / Rust SDK Tests (push) Blocked by required conditions
Details
Fly Deploy / Deploy app (push) Blocked by required conditions
Details
Fly Deploy / Build and publish Python SDK (push) Blocked by required conditions
Details
Fly Deploy / Build and publish JavaScript SDK (push) Blocked by required conditions
Details
Fly Deploy / Build and publish Rust SDK (push) Waiting to run
Details

Browse Source 
[Bug] Moved delete rawHtml to end of controller
This commit is contained in:
Nicolas 2024-08-30 12:40:06 -03:00 committed by GitHub
commit 2fd9c285bd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 21 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
apps/api/src/controllers/v0/crawl-status.ts
View File

@ -56,6 +56,19 @@ export async function crawlStatusController(req: Request, res: Response) {
const data = jobs.map(x => Array.isArray(x.returnvalue) ? x.returnvalue[0] : x.returnvalue); const data = jobs.map(x => Array.isArray(x.returnvalue) ? x.returnvalue[0] : x.returnvalue);
if (
jobs.length > 0 &&
jobs[0].data &&
jobs[0].data.pageOptions &&
!jobs[0].data.pageOptions.includeRawHtml
) {
data.forEach(item => {
if (item) {
delete item.rawHtml;
}
});
}
res.json({ res.json({
status: jobStatus, status: jobStatus,
current: jobStatuses.filter(x => x === "completed" || x === "failed").length, current: jobStatuses.filter(x => x === "completed" || x === "failed").length,

8
apps/api/src/controllers/v1/crawl-status.ts
View File

@ -100,6 +100,14 @@ export async function crawlStatusController(req: RequestWithAuth<CrawlStatusPara
nextURL.searchParams.set("limit", req.query.limit); nextURL.searchParams.set("limit", req.query.limit);
} }
if (data.length > 0) {
if (!doneJobs[0].data.pageOptions.includeRawHtml) {
for (let ii = 0; ii < doneJobs.length; ii++) {
delete data[ii].rawHtml;
}
}
}
res.status(200).json({ res.status(200).json({
status, status,
completed: doneJobsLength, completed: doneJobsLength,