Austin/reader
1
0
Fork 0
mirror of https://github.com/intergalacticalvariable/reader.git synced 2024-11-16 11:42:32 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix: apply rate limit to 100qpm per IP

Browse Source
This commit is contained in:
yanlong.wang 2024-04-29 18:54:51 +08:00
parent 15606f38d7
commit 867636d037
No known key found for this signature in database
GPG Key ID: C0A623C0BADF9F37
2 changed files with 11 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
backend/functions/src/cloud-functions/crawler.ts
View File

@ -6,6 +6,7 @@ import {
} from 'civkit';
import { singleton } from 'tsyringe';
import { CloudHTTPv2, Ctx, FirebaseStorageBucketControl, Logger, OutputServerEventStream, RPCReflect } from '../shared';
import { RateLimitControl } from '../shared/services/rate-limit';
import _ from 'lodash';
import { PageSnapshot, PuppeteerControl, ScrappingOptions } from '../services/puppeteer';
import { Request, Response } from 'express';
@ -36,6 +37,7 @@ export class CrawlerHost extends RPCHost {
protected puppeteerControl: PuppeteerControl,
protected altTextService: AltTextService,
protected firebaseObjectStorage: FirebaseStorageBucketControl,
protected rateLimitControl: RateLimitControl,
) {
super(...arguments);
@ -113,7 +115,7 @@ export class CrawlerHost extends RPCHost {
}
const toBeTurnedToMd = mode === 'markdown' ? snapshot.html : snapshot.parsed?.content;
let turnDownService = mode === 'markdown' ? this.getTurndown() : this.getTurndown('without any rule');
let turnDownService = mode === 'markdown' ? this.getTurndown() : this.getTurndown('without any rule');
for (const plugin of this.turnDownPlugins) {
turnDownService = turnDownService.use(plugin);
}
@ -295,6 +297,13 @@ ${this.content}
res: Response,
},
) {
if (ctx.req.ip) {
await this.rateLimitControl.simpleRpcIPBasedLimit(rpcReflect, ctx.req.ip, ['CRAWL'], [
// 100 requests per minute
new Date(Date.now() - 60 * 1000), 100
]);
}
const noSlashURL = ctx.req.url.slice(1);
if (!noSlashURL) {
return assignTransferProtocolMeta(`[Usage] https://r.jina.ai/YOUR_URL

2
thinapps-shared

@ -1 +1 @@
Subproject commit b165c1cb0e21b7b8762a23b8ce88219aa532c293
Subproject commit e681cf89bd21d77469dd286b2348e4cf5fce76e7