From 4c9d62f6d3c6cb7bd13590a8149e70dd81d8e282 Mon Sep 17 00:00:00 2001
From: Nicolas <nicolascamara29@gmail.com>
Date: Fri, 26 Jul 2024 18:25:44 -0400
Subject: [PATCH] Nick: fixing sitemap fallback

---
 apps/api/src/lib/entities.ts               | 1 +
 apps/api/src/scraper/WebScraper/sitemap.ts | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.ts
index 56cd793a..9ffa4810 100644
--- a/apps/api/src/lib/entities.ts
+++ b/apps/api/src/lib/entities.ts
@@ -139,4 +139,5 @@ export interface FireEngineOptions{
   engine?: string;
   blockMedia?: boolean;
   blockAds?: boolean;
+  disableJsDom?: boolean;
 }
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index b41f3eeb..3dfc9a1c 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -23,7 +23,7 @@ export async function getLinksFromSitemap(
         const response = await axios.get(sitemapUrl, { timeout: axiosTimeout });
         content = response.data;
       } else if (mode === 'fire-engine') {
-        const response = await scrapWithFireEngine({ url: sitemapUrl, fireEngineOptions: { method: "get", mobileProxy: true },options:{endpoint:"request"} });
+        const response = await scrapWithFireEngine({ url: sitemapUrl, fireEngineOptions: { engine:"tlsclient", disableJsDom: true, mobileProxy: true } });
         content = response.html;
       }
     } catch (error) {