From ee9a2184e2bdd39338bac058aa758af3dfe3600c Mon Sep 17 00:00:00 2001
From: rafaelsideguide <150964962+rafaelsideguide@users.noreply.github.com>
Date: Wed, 29 May 2024 13:39:43 -0300
Subject: [PATCH] Added custom scraping conditions for readme docs

---
 apps/api/src/scraper/WebScraper/single_url.ts | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts
index 70225048..4dfe79fa 100644
--- a/apps/api/src/scraper/WebScraper/single_url.ts
+++ b/apps/api/src/scraper/WebScraper/single_url.ts
@@ -210,6 +210,14 @@ function getScrapingFallbackOrder(defaultScraper?: string, isWaitPresent: boolea
   return scrapersInOrder as typeof baseScrapers[number][];
 }
 
+async function handleCustomScraping(text: string, url: string): Promise<string | null> {
+  if (text.includes('<meta name="readme-deploy"')) {
+    console.log(`Special use case detected for ${url}, using Fire Engine with wait time 1000ms`);
+    return await scrapWithFireEngine(url, 1000);
+  }
+  return null;
+}
+
 export async function scrapSingleUrl(
   urlToScrap: string,
   pageOptions: PageOptions = { onlyMainContent: true, includeHtml: false, waitFor: 0},
@@ -266,6 +274,12 @@ export async function scrapSingleUrl(
         break;
     }
 
+    // Check for custom scraping conditions
+    const customScrapedContent = await handleCustomScraping(text, url);
+    if (customScrapedContent) {
+      text = customScrapedContent;
+    }
+
     //* TODO: add an optional to return markdown or structured/extracted content
     let cleanedHtml = removeUnwantedElements(text, pageOptions);