Austin/reader
1
0
Fork 0
mirror of https://github.com/intergalacticalvariable/reader.git synced 2024-11-16 11:42:32 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor: Remove rate limiting from crawler.ts

Browse Source
This commit is contained in:
Harsh Gupta (aider) 2024-08-14 13:43:04 +05:30
parent aa862d4247
commit d15b721bfa
1 changed files with 2 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
backend/functions/src/cloud-functions/crawler.ts
View File

@ -3,6 +3,7 @@ import {
RPCHost, RPCReflection,
HashManager,
AssertionFailureError, ParamValidationError, Defer,
SecurityCompromiseError
} from 'civkit';
import { singleton } from 'tsyringe';
import { AsyncContext, CloudHTTPv2, Ctx, FirebaseStorageBucketControl, Logger, OutputServerEventStream, RPCReflect, SecurityCompromiseError } from '../shared';
@ -627,53 +628,7 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
ctx.req.hostname.toLowerCase()
);
if (uid) {
const user = await auth.assertUser();
if (!(user.wallet.total_balance > 0)) {
throw new InsufficientBalanceError(`Account balance not enough to run this query, please recharge.`);
}
const rateLimitPolicy = auth.getRateLimits(rpcReflect.name.toUpperCase()) || [
parseInt(user.metadata?.speed_level) >= 2 ?
RateLimitDesc.from({
occurrence: 1000,
periodSeconds: 60
}) :
RateLimitDesc.from({
occurrence: 200,
periodSeconds: 60
})
];
const apiRoll = await this.rateLimitControl.simpleRPCUidBasedLimit(
rpcReflect, uid, [rpcReflect.name.toUpperCase()],
...rateLimitPolicy
);
rpcReflect.finally(() => {
if (chargeAmount) {
auth.reportUsage(chargeAmount, `reader-${rpcReflect.name}`).catch((err) => {
this.logger.warn(`Unable to report usage for ${uid}`, { err: marshalErrorLike(err) });
});
apiRoll.chargeAmount = chargeAmount;
}
});
} else if (ctx.req.ip) {
const apiRoll = await this.rateLimitControl.simpleRpcIPBasedLimit(rpcReflect, ctx.req.ip, [rpcReflect.name.toUpperCase()],
[
// 20 requests per minute
new Date(Date.now() - 60 * 1000), 20
]
);
rpcReflect.finally(() => {
if (chargeAmount) {
apiRoll._ref?.set({
chargeAmount,
}, { merge: true }).catch((err) => this.logger.warn(`Failed to log charge amount in apiRoll`, { err }));
}
});
}
// Rate limiting code removed
let urlToCrawl;
const normalizeUrl = (await pNormalizeUrl).default;