From 15b774e9749f1dd644e88c7c735631876a0a12e3 Mon Sep 17 00:00:00 2001
From: Nicolas <nicolascamara29@gmail.com>
Date: Sat, 4 May 2024 12:44:30 -0700
Subject: [PATCH] Update index.ts

---
 apps/api/src/scraper/WebScraper/index.ts | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/apps/api/src/scraper/WebScraper/index.ts b/apps/api/src/scraper/WebScraper/index.ts
index 0cf001fe..1e285520 100644
--- a/apps/api/src/scraper/WebScraper/index.ts
+++ b/apps/api/src/scraper/WebScraper/index.ts
@@ -7,7 +7,6 @@ import { getValue, setValue } from "../../services/redis";
 import { getImageDescription } from "./utils/imageDescription";
 import { fetchAndProcessPdf } from "./utils/pdfProcessor";
 import { replaceImgPathsWithAbsolutePaths, replacePathsWithAbsolutePaths } from "./utils/replacePaths";
-import OpenAI from 'openai'
 import { generateCompletions } from "../../lib/LLM-extraction";
 
 
@@ -83,6 +82,11 @@ export class WebScraperDataProvider {
     }
   }
 
+  /**
+   * Process documents without cache handling each mode
+   * @param inProgress inProgress
+   * @returns documents
+   */
   private async processDocumentsWithoutCache(inProgress?: (progress: Progress) => void): Promise<Document[]> {
     switch (this.mode) {
       case "crawl":