Nick: set the crawl limit to the remaining credits

apps/api/src/controllers/crawl.ts

@@ -56,14 +56,17 @@ export async function crawlController(req: Request, res: Response) {
     };
     const pageOptions = { ...defaultCrawlPageOptions, ...req.body.pageOptions };
 
-    const limitCheck = crawlerOptions?.limit ?? 1;
+    const limitCheck = req.body?.crawlerOptions?.limit ?? 1;
-    const { success: creditsCheckSuccess, message: creditsCheckMessage } =
+    const { success: creditsCheckSuccess, message: creditsCheckMessage, remainingCredits } =
       await checkTeamCredits(team_id, limitCheck);
 
     if (!creditsCheckSuccess) {
-      return res.status(402).json({ error: "Insufficient credits" });
+      return res.status(402).json({ error: "Insufficient credits. You may be requesting with a higher limit than the amount of credits you have left. If not, upgrade your plan at https://firecrawl.dev/pricing or contact us at hello@firecrawl.com" });
     }
 
+    // TODO: need to do this to v1
+    crawlerOptions.limit = Math.min(remainingCredits, crawlerOptions.limit);
+    
     let url = req.body.url;
     if (!url) {
       return res.status(400).json({ error: "Url is required" });

apps/api/src/lib/default-values.ts

@@ -12,7 +12,8 @@ export const defaultPageOptions = {
 };
 
 export const defaultCrawlerOptions = {
-  allowBackwardCrawling: false
+  allowBackwardCrawling: false,
+  limit: 10000
 }
 
 export const defaultCrawlPageOptions = {

apps/api/src/services/billing/credit_billing.ts

@@ -168,10 +168,11 @@ export async function supaBillTeam(team_id: string, credits: number) {
 export async function checkTeamCredits(team_id: string, credits: number) {
   return withAuth(supaCheckTeamCredits)(team_id, credits);
 }
+
 // if team has enough credits for the operation, return true, else return false
 export async function supaCheckTeamCredits(team_id: string, credits: number) {
   if (team_id === "preview") {
-    return { success: true, message: "Preview team, no credits used" };
+    return { success: true, message: "Preview team, no credits used", remainingCredits: Infinity };
   }
 
   // Retrieve the team's active subscription and check for available coupons concurrently
@@ -202,7 +203,7 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) {
   if (subscriptionError || !subscription) {
     // If there is no active subscription but there are available coupons
     if (couponCredits >= credits) {
-      return { success: true, message: "Sufficient credits available" };
+      return { success: true, message: "Sufficient credits available", remainingCredits: couponCredits };
     }
 
     const { data: creditUsages, error: creditUsageError } =
@@ -252,9 +253,10 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) {
       return {
         success: false,
         message: "Insufficient credits, please upgrade!",
+        remainingCredits: FREE_CREDITS - totalCreditsUsed
       };
     }
-    return { success: true, message: "Sufficient credits available" };
+    return { success: true, message: "Sufficient credits available", remainingCredits: FREE_CREDITS - totalCreditsUsed };
   }
 
   let totalCreditsUsed = 0;
@@ -321,7 +323,7 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) {
       subscription.current_period_start,
       subscription.current_period_end
     );
-    return { success: false, message: "Insufficient credits, please upgrade!" };
+    return { success: false, message: "Insufficient credits, please upgrade!", remainingCredits: creditLimit - adjustedCreditsUsed };
   } else if (creditUsagePercentage >= 0.8) {
     // Send email notification for approaching credit limit
     await sendNotification(
@@ -332,7 +334,7 @@ export async function supaCheckTeamCredits(team_id: string, credits: number) {
     );
   }
 
-  return { success: true, message: "Sufficient credits available" };
+  return { success: true, message: "Sufficient credits available", remainingCredits: creditLimit - adjustedCreditsUsed };
 }
 
 // Count the total credits used by a team within the current billing period and return the remaining credits.

apps/api/src/services/queue-worker.ts

@@ -117,13 +117,13 @@ async function processJob(job: Job, token: string) {
 
   // Check if the job URL is researchhub and block it immediately
   // TODO: remove this once solve the root issue
-  if (job.data.url && (job.data.url.includes("researchhub.com") || job.data.url.includes("ebay.com") || job.data.url.includes("youtube.com"))) {
+  if (job.data.url && (job.data.url.includes("researchhub.com") || job.data.url.includes("ebay.com") || job.data.url.includes("youtube.com") || job.data.url.includes("microsoft.com"))) {
     Logger.info(`🐂 Blocking job ${job.id} with URL ${job.data.url}`);
     const data = {
       success: false,
       docs: [],
       project_id: job.data.project_id,
-      error: "URL is blocked. Please contact hello@firecrawl.com if you believe this is an error.",
+      error: "URL is blocked. Suspecious activity detected. Please contact hello@firecrawl.com if you believe this is an error.",
     };
     await job.moveToCompleted(data.docs, token, false);
     return data;