Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 19:58:08 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Nick: set the crawl limit to the remaining credits

Browse Source
This commit is contained in:
Nicolas 2024-08-20 14:16:54 -03:00
parent e1c9cbf709
commit 5abd26a267
4 changed files with 17 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
apps/api/src/controllers/crawl.ts
View File

@ -56,14 +56,17 @@ export async function crawlController(req: Request, res: Response) {
};
const pageOptions = { ...defaultCrawlPageOptions, ...req.body.pageOptions };
const limitCheck = crawlerOptions?.limit ?? 1;
const { success: creditsCheckSuccess, message: creditsCheckMessage } =
const limitCheck = req.body?.crawlerOptions?.limit ?? 1;
const { success: creditsCheckSuccess, message: creditsCheckMessage, remainingCredits } =
await checkTeamCredits(team_id, limitCheck);
if (!creditsCheckSuccess) {
return res.status(402).json({ error: "Insufficient credits" });
return res.status(402).json({ error: "Insufficient credits. You may be requesting with a higher limit than the amount of credits you have left. If not, upgrade your plan at https://firecrawl.dev/pricing or contact us at hello@firecrawl.com" });
}
// TODO: need to do this to v1
crawlerOptions.limit = Math.min(remainingCredits, crawlerOptions.limit);
let url = req.body.url;
if (!url) {
return res.status(400).json({ error: "Url is required" });

3
apps/api/src/lib/default-values.ts
View File

@ -12,7 +12,8 @@ export const defaultPageOptions = {
};
export const defaultCrawlerOptions = {
allowBackwardCrawling: false
allowBackwardCrawling: false,
limit: 10000
}
export const defaultCrawlPageOptions = {

12
apps/api/src/services/billing/credit_billing.ts
View File

@ -168,10 +168,11 @@ export async function supaBillTeam(team_id: string, credits: number) {
export async function checkTeamCredits(team_id: string, credits: number) {
return withAuth(supaCheckTeamCredits)(team_id, credits);
}
// if team has enough credits for the operation, return true, else return false
export async function supaCheckTeamCredits(team_id: string, credits: number) {
if (team_id === "preview") {
return { success: true, message: "Preview team, no credits used" };
return { success: true, message: "Preview team, no credits used", remainingCredits: Infinity };
}
// Retrieve the team's active subscription and check for available coupons concurrently
@ -202,7 +203,7 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) {
if (subscriptionError || !subscription) {
// If there is no active subscription but there are available coupons
if (couponCredits >= credits) {
return { success: true, message: "Sufficient credits available" };
return { success: true, message: "Sufficient credits available", remainingCredits: couponCredits };
}
const { data: creditUsages, error: creditUsageError } =
@ -252,9 +253,10 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) {
return {
success: false,
message: "Insufficient credits, please upgrade!",
remainingCredits: FREE_CREDITS - totalCreditsUsed
};
}
return { success: true, message: "Sufficient credits available" };
return { success: true, message: "Sufficient credits available", remainingCredits: FREE_CREDITS - totalCreditsUsed };
}
let totalCreditsUsed = 0;
@ -321,7 +323,7 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) {
subscription.current_period_start,
subscription.current_period_end
);
return { success: false, message: "Insufficient credits, please upgrade!" };
return { success: false, message: "Insufficient credits, please upgrade!", remainingCredits: creditLimit - adjustedCreditsUsed };
} else if (creditUsagePercentage >= 0.8) {
// Send email notification for approaching credit limit
await sendNotification(
@ -332,7 +334,7 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) {
);
}
return { success: true, message: "Sufficient credits available" };
return { success: true, message: "Sufficient credits available", remainingCredits: creditLimit - adjustedCreditsUsed };
}
// Count the total credits used by a team within the current billing period and return the remaining credits.

4
apps/api/src/services/queue-worker.ts
View File

@ -117,13 +117,13 @@ async function processJob(job: Job, token: string) {
// Check if the job URL is researchhub and block it immediately
// TODO: remove this once solve the root issue
if (job.data.url && (job.data.url.includes("researchhub.com") || job.data.url.includes("ebay.com") || job.data.url.includes("youtube.com"))) {
if (job.data.url && (job.data.url.includes("researchhub.com") || job.data.url.includes("ebay.com") || job.data.url.includes("youtube.com") || job.data.url.includes("microsoft.com"))) {
Logger.info(`🐂 Blocking job ${job.id} with URL ${job.data.url}`);
const data = {
success: false,
docs: [],
project_id: job.data.project_id,
error: "URL is blocked. Please contact hello@firecrawl.com if you believe this is an error.",
error: "URL is blocked. Suspecious activity detected. Please contact hello@firecrawl.com if you believe this is an error.",
};
await job.moveToCompleted(data.docs, token, false);
return data;