From 66e5d42d2e10b7fe98d9787c461fed46e2b48d6f Mon Sep 17 00:00:00 2001 From: Harsh Gupta Date: Wed, 18 Sep 2024 09:48:30 +0530 Subject: [PATCH] cleanup --- backend/functions/src/services/puppeteer.ts | 28 +++------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/backend/functions/src/services/puppeteer.ts b/backend/functions/src/services/puppeteer.ts index 77d5107..12fa15b 100644 --- a/backend/functions/src/services/puppeteer.ts +++ b/backend/functions/src/services/puppeteer.ts @@ -15,7 +15,7 @@ import tldExtract from 'tld-extract'; // Add this new function for cookie validation const validateCookie = (cookie: CookieParam) => { - const requiredFields = ['name', 'value', 'domain']; + const requiredFields = ['name', 'value']; for (const field of requiredFields) { if (!cookie[field]) { throw new Error(`Cookie is missing required field: ${field}`); @@ -482,12 +482,12 @@ document.addEventListener('load', handlePageLoad); const page = await this.getNextPage(); const sn = this.snMap.get(page); this.logger.info(`Page ${sn}: Scraping ${url}`, { url }); - + if (options?.proxyUrl) { this.logger.info(`Page ${sn}: Using proxy:`, options.proxyUrl); await page.useProxy(options.proxyUrl); } - + if (options?.cookies) { this.logger.info(`Page ${sn}: Attempting to set cookies:`, JSON.stringify(options.cookies, null, 2)); try { @@ -499,7 +499,7 @@ document.addEventListener('load', handlePageLoad); throw error; } } - + if (options?.overrideUserAgent) { await page.setUserAgent(options.overrideUserAgent); } @@ -614,26 +614,6 @@ document.addEventListener('load', handlePageLoad); } }); - try { - console.log('Waiting for Sign Out link'); - await page.waitForSelector('a[href="/signout"]', { timeout: 10000 }); - console.log('Sign Out link found'); - } catch (error) { - this.logger.warn(`Timed out waiting for Sign Out link: ${error}`); - } - - try { - console.log('Clicking on #getOwnerDetails'); - await page.click('#getOwnerDetails'); - console.log('Clicked on #getOwnerDetails'); - - // console.log('Waiting for Phone No span to appear'); - // await page.waitForSelector('span:contains("Phone No :")', { timeout: 10000 }); - // console.log('Phone No span appeared'); - } catch (error) { - this.logger.warn(`Error during click or wait operation: ${error}`); - } - let waitForPromise: Promise | undefined; if (options?.waitForSelector) { console.log('Waiting for selector', options.waitForSelector);