Austin/reader
1
0
Fork 0
mirror of https://github.com/intergalacticalvariable/reader.git synced 2024-11-16 11:42:32 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: Modify crawler.ts and index.ts to make crawl function usable as a Firebase function

Browse Source
This commit is contained in:
Harsh Gupta (aider) 2024-08-14 14:30:08 +05:30
parent 80547abf38
commit 2343c1d28b
2 changed files with 12 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
backend/functions/src/cloud-functions/crawler.ts
View File

@ -566,41 +566,11 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
return formatted as FormattedPage;
}
@CloudHTTPv2({
name: 'crawl2',
runtime: {
memory: '4GiB',
timeoutSeconds: 300,
concurrency: 22,
},
tags: ['Crawler'],
httpMethod: ['get', 'post'],
returnType: [String, OutputServerEventStream],
exposeRoot: true,
})
@CloudHTTPv2({
runtime: {
memory: '4GiB',
cpu: 4,
timeoutSeconds: 300,
concurrency: 22,
maxInstances: 455,
minInstances: 1,
},
tags: ['Crawler'],
httpMethod: ['get', 'post'],
returnType: [String, OutputServerEventStream],
exposeRoot: true,
})
async crawl(
@RPCReflect() rpcReflect: RPCReflection,
ctx: {
req: Request,
res: Response,
},
crawlerOptionsHeaderOnly: CrawlerOptionsHeaderOnly,
crawlerOptionsParamsAllowed: CrawlerOptions,
) {
async crawl(req: Request, res: Response) {
const rpcReflect = new RPCReflection();
const ctx = { req, res };
const crawlerOptionsHeaderOnly = CrawlerOptionsHeaderOnly.from(req.headers);
const crawlerOptionsParamsAllowed = CrawlerOptions.from(req.method === 'POST' ? req.body : req.query);
const noSlashURL = ctx.req.url.slice(1);
const crawlerOptions = ctx.req.method === 'GET' ? crawlerOptionsHeaderOnly : crawlerOptionsParamsAllowed;
if (!noSlashURL && !crawlerOptions.url) {
@ -887,4 +857,4 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
return this.formatSnapshot(mode, lastSnapshot, url);
}
}
}

7
backend/functions/src/index.ts
View File

@ -7,7 +7,12 @@ import { https } from 'firebase-functions'
initializeApp();
export const crawler = https.onRequest(CrawlerHost);
import { container } from 'tsyringe';
const crawlerHost = container.resolve(CrawlerHost);
export const crawler = https.onRequest(async (req, res) => {
await crawlerHost.crawl(req, res);
});
// import { loadModulesDynamically, registry } from './shared';
// import path from 'path';
// loadModulesDynamically(path.resolve(__dirname, 'cloud-functions'));