Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

fix(queue-worker): variable name collision

Browse Source
This commit is contained in:
Gergő Móricz 2024-08-14 22:02:05 +02:00
parent b8ec40dd72
commit 8a5cad72f6
1 changed files with 7 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
apps/api/src/services/queue-worker.ts
View File

@ -175,7 +175,7 @@ async function processJob(job: Job, token: string) {
const crawler = crawlToCrawler(job.data.crawl_id, sc);
const links = crawler.filterLinks((data.docs[0].linksOnPage as string[])
.map(href => crawler.filterURL(href, sc.originUrl))
.map(href => crawler.filterURL(href.trim(), sc.originUrl))
.filter(x => x !== null),
Infinity,
sc.crawlerOptions?.maxDepth ?? 10
@ -228,12 +228,12 @@ async function processJob(job: Job, token: string) {
const jobStatuses = await Promise.all(jobs.map(x => x.getState()));
const jobStatus = sc.cancelled ? "failed" : jobStatuses.every(x => x === "completed") ? "completed" : jobStatuses.some(x => x === "failed") ? "failed" : "active";
const docs = jobs.map(x => Array.isArray(x.returnvalue) ? x.returnvalue[0] : x.returnvalue);
const fullDocs = jobs.map(x => Array.isArray(x.returnvalue) ? x.returnvalue[0] : x.returnvalue);
const data = {
success: jobStatus !== "failed",
result: {
links: docs.map((doc) => {
links: fullDocs.map((doc) => {
return {
content: doc,
source: doc?.metadata?.sourceURL ?? doc?.url ?? "",
@ -242,7 +242,7 @@ async function processJob(job: Job, token: string) {
},
project_id: job.data.project_id,
error: message /* etc... */,
docs,
docs: fullDocs,
};
await callWebhook(job.data.team_id, job.id as string, data);
@ -265,6 +265,9 @@ async function processJob(job: Job, token: string) {
});
}
Logger.error(error);
if (error.stack) {
Logger.error(error.stack);
}
logtail.error("Overall error ingesting", {
job_id: job.id,