Austin/reader
1
0
Fork 0
mirror of https://github.com/intergalacticalvariable/reader.git synced 2024-11-16 11:42:32 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
Yanlong Wang 2024-04-13 07:53:58 +08:00
parent be7eeec11b
commit 5ed3f90b9c
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
3 changed files with 15 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/functions/.puppeteerrc.cjs
View File

@ -4,7 +4,7 @@ let config = {};
if (!process.env.FUNCTIONS_EMULATOR) {
config = {
// Changes the cache location for Puppeteer.
cacheDirectory: join(__dirname, '.cache', 'puppeteer'),
cacheDirectory: join(__dirname, 'puppeteer'),
};
}

4
backend/functions/src/cloud-functions/crawler.ts
View File

@ -65,9 +65,9 @@ ${this.content}
})
@CloudHTTPv2({
runtime: {
memory: '4GiB',
memory: '8GiB',
timeoutSeconds: 540,
concurrency: 4,
concurrency: 16,
},
httpMethod: ['get', 'post'],
returnType: [String, OutputServerEventStream],

15
backend/functions/src/services/puppeteer.ts
View File

@ -49,11 +49,12 @@ export class PuppeteerControl extends AsyncService {
return page.browser().connected && !page.isClosed();
}
}, {
max: Math.max(1 + Math.floor(os.freemem() / 1024 * 1024 * 1024), 4),
max: Math.max(1 + Math.floor(os.freemem() / 1024 * 1024 * 1024), 16),
min: 1,
acquireTimeoutMillis: 60_000,
testOnBorrow: true,
testOnReturn: true,
autostart: false,
});
constructor(protected globalLogger: Logger) {
@ -63,6 +64,8 @@ export class PuppeteerControl extends AsyncService {
override async init() {
await this.dependencyReady();
this.pagePool.start();
if (this.browser) {
if (this.browser.connected) {
await this.browser.close();
@ -72,7 +75,13 @@ export class PuppeteerControl extends AsyncService {
}
this.browser = await puppeteer.launch({
headless: true,
timeout: 45_000
timeout: 10_000
}).catch((err) => {
this.logger.error(`Unknown firebase issue, just die fast, quitting process.`, { err });
process.nextTick(()=> {
process.exit(1);
});
return Promise.reject(err);
});
this.browser.once('disconnected', () => {
this.logger.warn(`Browser disconnected`);
@ -90,7 +99,7 @@ export class PuppeteerControl extends AsyncService {
const page = await dedicatedContext.newPage();
const preparations = [];
preparations.push(page.setUserAgent(`Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)`));
// preparations.push(page.setUserAgent(`Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)`));
preparations.push(page.setBypassCSP(true));
preparations.push(page.setViewport({ width: 1920, height: 1080 }));
preparations.push(page.exposeFunction('reportSnapshot', (snapshot: any) => {