diff --git a/apps/api/src/lib/scrape-events.ts b/apps/api/src/lib/scrape-events.ts index cd8cfa9a..8d677279 100644 --- a/apps/api/src/lib/scrape-events.ts +++ b/apps/api/src/lib/scrape-events.ts @@ -59,12 +59,6 @@ export class ScrapeEvents { try { const previousLog = (await supabase.from("scrape_events").select().eq("id", logId).single()).data as any; - - if (!previousLog) { - Logger.warn("Previous log not found."); - return; - } - await supabase.from("scrape_events").update({ content: { ...previousLog.content, diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts index ba22f28b..76665aa2 100644 --- a/apps/api/src/main/runWebScraper.ts +++ b/apps/api/src/main/runWebScraper.ts @@ -131,7 +131,7 @@ const saveJob = async (job: Job, result: any) => { if (error) throw new Error(error.message); try { - // await job.moveToCompleted(null, false, false); + await job.moveToCompleted(null, false, false); } catch (error) { // I think the job won't exist here anymore } diff --git a/apps/api/src/scraper/WebScraper/index.ts b/apps/api/src/scraper/WebScraper/index.ts index c3834bcd..e667fa6b 100644 --- a/apps/api/src/scraper/WebScraper/index.ts +++ b/apps/api/src/scraper/WebScraper/index.ts @@ -94,7 +94,7 @@ export class WebScraperDataProvider { const jobStatus = await job.getState(); if (jobStatus === "failed") { Logger.info( - "Job " + job.id + " has failed or has been cancelled by the user. Stopping the job..." + "Job has failed or has been cancelled by the user. Stopping the job..." ); return [] as Document[]; } diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.ts index d531c2db..0cd65f32 100644 --- a/apps/api/src/services/queue-service.ts +++ b/apps/api/src/services/queue-service.ts @@ -14,7 +14,7 @@ export function getWebScraperQueue() { maxStalledCount: 10, }, defaultJobOptions:{ - attempts: 5 + attempts: 2 } }); Logger.info("Web scraper queue created"); diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts index bddf9300..cc92b3ab 100644 --- a/apps/api/src/services/queue-worker.ts +++ b/apps/api/src/services/queue-worker.ts @@ -22,11 +22,6 @@ const wsq = getWebScraperQueue(); async function processJob(job: Job, done) { Logger.info(`🐂 Worker taking job ${job.id}`); - const lockInterval = setInterval(() => { - Logger.info(`🐂 Renewing lock for ${job.id}`); - job.extendLock(60000); - }, 15000); - try { job.progress({ current: 1, @@ -67,8 +62,7 @@ async function processJob(job: Job, done) { origin: job.data.origin, }); Logger.info(`🐂 Job done ${job.id}`); - clearInterval(lockInterval); - done(null, null); + done(null, data); } catch (error) { Logger.error(`🐂 Job errored ${job.id} - ${error}`); if (await getWebScraperQueue().isPaused(false)) { @@ -114,9 +108,8 @@ async function processJob(job: Job, done) { pageOptions: job.data.pageOptions, origin: job.data.origin, }); - clearInterval(lockInterval); done(null, data); - } + } } wsq.process(