From 91f52287dbf2a2126af10b93ba6680397c93e534 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?M=C3=B3ricz=20Gerg=C5=91?= <mo.geryy@gmail.com>
Date: Tue, 12 Nov 2024 12:42:39 +0100
Subject: [PATCH] feat(batchScrape): handle timeout

---
 apps/api/src/services/queue-worker.ts | 34 +++++++++++++++++++--------
 1 file changed, 24 insertions(+), 10 deletions(-)

diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 21dd472e..0959424a 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -276,10 +276,18 @@ async function processJob(job: Job & { id: string }, token: string) {
     });
     const start = Date.now();
 
-    const pipeline = await startWebScraperPipeline({
-      job,
-      token,
-    });
+    const pipeline = await Promise.race([
+      startWebScraperPipeline({
+        job,
+        token,
+      }),
+      ...(job.data.scrapeOptions.timeout !== undefined ? [
+        (async () => {
+          await sleep(job.data.scrapeOptions.timeout);
+          throw new Error("timeout")
+        })(),
+      ] : [])
+    ]);
 
     if (!pipeline.success) {
       // TODO: let's Not do this
@@ -486,13 +494,19 @@ async function processJob(job: Job & { id: string }, token: string) {
     logger.info(`🐂 Job done ${job.id}`);
     return data;
   } catch (error) {
-    logger.error(`🐂 Job errored ${job.id} - ${error}`);
+    const isEarlyTimeout = error instanceof Error && error.message === "timeout";
 
-    Sentry.captureException(error, {
-      data: {
-        job: job.id,
-      },
-    });
+    if (!isEarlyTimeout) {
+      logger.error(`🐂 Job errored ${job.id} - ${error}`);
+
+      Sentry.captureException(error, {
+        data: {
+          job: job.id,
+        },
+      });
+    } else {
+      logger.error(`🐂 Job timed out ${job.id}`);
+    }
 
     if (error instanceof CustomError) {
       // Here we handle the error, then save the failed job