Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

feat(queue-worker): always crawl links from content even if sitemapped

Browse Source
This commit is contained in:
Gergő Móricz 2024-10-01 18:32:53 +02:00
parent 1af26fe1b4
commit 3c045c43a4
1 changed files with 31 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
apps/api/src/services/queue-worker.ts
View File

@ -300,7 +300,6 @@ async function processJob(job: Job, token: string) {
const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
if (!job.data.sitemapped) {
if (!sc.cancelled) {
const crawler = crawlToCrawler(job.data.crawl_id, sc);
@ -312,7 +311,6 @@ async function processJob(job: Job, token: string) {
for (const link of links) {
if (await lockURL(job.data.crawl_id, sc, link)) {
// This seems to work really welel
const jobPriority = await getJobPriority({
plan: sc.plan as PlanType,
team_id: sc.team_id,
@ -320,11 +318,6 @@ async function processJob(job: Job, token: string) {
});
const jobId = uuidv4();
// console.log("plan: ", sc.plan);
// console.log("team_id: ", sc.team_id)
// console.log("base priority: ", job.data.crawl_id ? 20 : 10)
// console.log("job priority: " , jobPriority, "\n\n\n")
const newJob = await addScrapeJob(
{
url: link,
@ -345,7 +338,6 @@ async function processJob(job: Job, token: string) {
}
}
}
}
if (await finishCrawl(job.data.crawl_id)) {