Austin/reader
1
0
Fork 0
mirror of https://github.com/intergalacticalvariable/reader.git synced 2024-11-16 11:42:32 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
Yanlong Wang 2024-04-13 07:53:58 +08:00
parent be7eeec11b
commit 5ed3f90b9c
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
3 changed files with 15 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/functions/.puppeteerrc.cjs
View File

@ -4,7 +4,7 @@ let config = {};
if (!process.env.FUNCTIONS_EMULATOR) { if (!process.env.FUNCTIONS_EMULATOR) {
config = { config = {
// Changes the cache location for Puppeteer. // Changes the cache location for Puppeteer.
cacheDirectory: join(__dirname, '.cache', 'puppeteer'), cacheDirectory: join(__dirname, 'puppeteer'),
}; };
} }

4
backend/functions/src/cloud-functions/crawler.ts
View File

@ -65,9 +65,9 @@ ${this.content}
}) })
@CloudHTTPv2({ @CloudHTTPv2({
runtime: { runtime: {
memory: '4GiB', memory: '8GiB',
timeoutSeconds: 540, timeoutSeconds: 540,
concurrency: 4, concurrency: 16,
}, },
httpMethod: ['get', 'post'], httpMethod: ['get', 'post'],
returnType: [String, OutputServerEventStream], returnType: [String, OutputServerEventStream],

15
backend/functions/src/services/puppeteer.ts
View File

@ -49,11 +49,12 @@ export class PuppeteerControl extends AsyncService {
return page.browser().connected && !page.isClosed(); return page.browser().connected && !page.isClosed();
} }
}, { }, {
max: Math.max(1 + Math.floor(os.freemem() / 1024 * 1024 * 1024), 4), max: Math.max(1 + Math.floor(os.freemem() / 1024 * 1024 * 1024), 16),
min: 1, min: 1,
acquireTimeoutMillis: 60_000, acquireTimeoutMillis: 60_000,
testOnBorrow: true, testOnBorrow: true,
testOnReturn: true, testOnReturn: true,
autostart: false,
}); });
constructor(protected globalLogger: Logger) { constructor(protected globalLogger: Logger) {
@ -63,6 +64,8 @@ export class PuppeteerControl extends AsyncService {
override async init() { override async init() {
await this.dependencyReady(); await this.dependencyReady();
this.pagePool.start();
if (this.browser) { if (this.browser) {
if (this.browser.connected) { if (this.browser.connected) {
await this.browser.close(); await this.browser.close();
@ -72,7 +75,13 @@ export class PuppeteerControl extends AsyncService {
} }
this.browser = await puppeteer.launch({ this.browser = await puppeteer.launch({
headless: true, headless: true,
timeout: 45_000 timeout: 10_000
}).catch((err) => {
this.logger.error(`Unknown firebase issue, just die fast, quitting process.`, { err });
process.nextTick(()=> {
process.exit(1);
});
return Promise.reject(err);
}); });
this.browser.once('disconnected', () => { this.browser.once('disconnected', () => {
this.logger.warn(`Browser disconnected`); this.logger.warn(`Browser disconnected`);
@ -90,7 +99,7 @@ export class PuppeteerControl extends AsyncService {
const page = await dedicatedContext.newPage(); const page = await dedicatedContext.newPage();
const preparations = []; const preparations = [];
preparations.push(page.setUserAgent(`Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)`)); // preparations.push(page.setUserAgent(`Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)`));
preparations.push(page.setBypassCSP(true)); preparations.push(page.setBypassCSP(true));
preparations.push(page.setViewport({ width: 1920, height: 1080 })); preparations.push(page.setViewport({ width: 1920, height: 1080 }));
preparations.push(page.exposeFunction('reportSnapshot', (snapshot: any) => { preparations.push(page.exposeFunction('reportSnapshot', (snapshot: any) => {