Austin/reader
1
0
Fork 0
mirror of https://github.com/intergalacticalvariable/reader.git synced 2024-11-16 11:42:32 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

cleanup

Browse Source
This commit is contained in:
Harsh Gupta 2024-09-18 09:48:30 +05:30
parent cc4d316764
commit 66e5d42d2e
1 changed files with 4 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
backend/functions/src/services/puppeteer.ts
View File

@ -15,7 +15,7 @@ import tldExtract from 'tld-extract';
// Add this new function for cookie validation // Add this new function for cookie validation
const validateCookie = (cookie: CookieParam) => { const validateCookie = (cookie: CookieParam) => {
const requiredFields = ['name', 'value', 'domain']; const requiredFields = ['name', 'value'];
for (const field of requiredFields) { for (const field of requiredFields) {
if (!cookie[field]) { if (!cookie[field]) {
throw new Error(`Cookie is missing required field: ${field}`); throw new Error(`Cookie is missing required field: ${field}`);
@ -482,12 +482,12 @@ document.addEventListener('load', handlePageLoad);
const page = await this.getNextPage(); const page = await this.getNextPage();
const sn = this.snMap.get(page); const sn = this.snMap.get(page);
this.logger.info(`Page ${sn}: Scraping ${url}`, { url }); this.logger.info(`Page ${sn}: Scraping ${url}`, { url });
if (options?.proxyUrl) { if (options?.proxyUrl) {
this.logger.info(`Page ${sn}: Using proxy:`, options.proxyUrl); this.logger.info(`Page ${sn}: Using proxy:`, options.proxyUrl);
await page.useProxy(options.proxyUrl); await page.useProxy(options.proxyUrl);
} }
if (options?.cookies) { if (options?.cookies) {
this.logger.info(`Page ${sn}: Attempting to set cookies:`, JSON.stringify(options.cookies, null, 2)); this.logger.info(`Page ${sn}: Attempting to set cookies:`, JSON.stringify(options.cookies, null, 2));
try { try {
@ -499,7 +499,7 @@ document.addEventListener('load', handlePageLoad);
throw error; throw error;
} }
} }
if (options?.overrideUserAgent) { if (options?.overrideUserAgent) {
await page.setUserAgent(options.overrideUserAgent); await page.setUserAgent(options.overrideUserAgent);
} }
@ -614,26 +614,6 @@ document.addEventListener('load', handlePageLoad);
} }
}); });
try {
console.log('Waiting for Sign Out link');
await page.waitForSelector('a[href="/signout"]', { timeout: 10000 });
console.log('Sign Out link found');
} catch (error) {
this.logger.warn(`Timed out waiting for Sign Out link: ${error}`);
}
try {
console.log('Clicking on #getOwnerDetails');
await page.click('#getOwnerDetails');
console.log('Clicked on #getOwnerDetails');
// console.log('Waiting for Phone No span to appear');
// await page.waitForSelector('span:contains("Phone No :")', { timeout: 10000 });
// console.log('Phone No span appeared');
} catch (error) {
this.logger.warn(`Error during click or wait operation: ${error}`);
}
let waitForPromise: Promise<any> | undefined; let waitForPromise: Promise<any> | undefined;
if (options?.waitForSelector) { if (options?.waitForSelector) {
console.log('Waiting for selector', options.waitForSelector); console.log('Waiting for selector', options.waitForSelector);