Austin/reader
1
0
Fork 0
mirror of https://github.com/intergalacticalvariable/reader.git synced 2024-11-16 11:42:32 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: Update crawler.ts to log cookies

Browse Source
This commit is contained in:
Harsh Gupta 2024-08-15 22:43:49 +05:30 committed by Harsh Gupta (aider)
parent 1a5f2eb408
commit dcc3e5294d
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
backend/functions/src/cloud-functions/crawler.ts
View File

@ -877,7 +877,7 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
this.threadLocal.set('timeout', opts.timeout * 1000); this.threadLocal.set('timeout', opts.timeout * 1000);
} }
const cookies = req.headers['x-set-cookie'] ? const cookies = req.headers['x-set-cookie'] ?
(Array.isArray(req.headers['x-set-cookie']) ? req.headers['x-set-cookie'] : [req.headers['x-set-cookie']]) (Array.isArray(req.headers['x-set-cookie']) ? req.headers['x-set-cookie'] : [req.headers['x-set-cookie']])
.map(cookie => { .map(cookie => {
const [name, value] = cookie.split('='); const [name, value] = cookie.split('=');
@ -885,6 +885,7 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
}) })
: []; : [];
console.log('Cookies:', cookies);
const crawlOpts: ExtraScrappingOptions = { const crawlOpts: ExtraScrappingOptions = {
proxyUrl: opts.proxyUrl, proxyUrl: opts.proxyUrl,
cookies: cookies, cookies: cookies,