Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Nick: cleaner functions to handle authenticated requests that dont require ifs everywhere

Browse Source
This commit is contained in:
Nicolas 2024-04-21 10:36:48 -07:00
parent aa89e2e8b5
commit 5cdbf3a0ac
7 changed files with 76 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
apps/api/src/controllers/auth.ts
View File

@ -1,10 +1,15 @@
import { parseApi } from "../../src/lib/parseApi";
import { getRateLimiter } from "../../src/services/rate-limiter";
import { RateLimiterMode } from "../../src/types";
import { AuthResponse, RateLimiterMode } from "../../src/types";
import { supabase_service } from "../../src/services/supabase";
import { withAuth } from "../../src/lib/withAuth";
export async function authenticateUser(
export async function authenticateUser(req, res, mode?: RateLimiterMode) : Promise<AuthResponse> {
return withAuth(supaAuthenticateUser)(req, res, mode);
}
export async function supaAuthenticateUser(
req,
res,
mode?: RateLimiterMode
@ -15,15 +20,6 @@ export async function authenticateUser(
status?: number;
}> {
console.log(process.env)
if(process.env.USE_DB_AUTHENTICATION === "false"){
console.log("WARNING - YOU'RE bypassing Authentication");
return { success: true};
}
console.log("USING SUPABASE AUTH");
const authHeader = req.headers.authorization;
if (!authHeader) {
return { success: false, error: "Unauthorized", status: 401 };

16
apps/api/src/controllers/crawl.ts
View File

@ -8,8 +8,7 @@ import { addWebScraperJob } from "../../src/services/queue-jobs";
export async function crawlController(req: Request, res: Response) {
try {
console.log("hello")
console.log("hello");
const { success, team_id, error, status } = await authenticateUser(
req,
res,
@ -18,13 +17,11 @@ export async function crawlController(req: Request, res: Response) {
if (!success) {
return res.status(status).json({ error });
}
if (process.env.USE_DB_AUTHENTICATION === "true") {
const { success: creditsCheckSuccess, message: creditsCheckMessage } =
await checkTeamCredits(team_id, 1);
if (!creditsCheckSuccess) {
return res.status(402).json({ error: "Insufficient credits" });
}
const { success: creditsCheckSuccess, message: creditsCheckMessage } =
await checkTeamCredits(team_id, 1);
if (!creditsCheckSuccess) {
return res.status(402).json({ error: "Insufficient credits" });
}
const url = req.body.url;
@ -45,7 +42,6 @@ export async function crawlController(req: Request, res: Response) {
returnOnlyUrls: true,
},
pageOptions: pageOptions,
});
const docs = await a.getDocuments(false, (progress) => {

2
apps/api/src/controllers/scrape.ts
View File

@ -41,7 +41,6 @@ export async function scrapeHelper(
return { success: true, error: "No page found", returnCode: 200 };
}
if (process.env.USE_DB_AUTHENTICATION === "true") {
const { success, credit_usage } = await billTeam(
team_id,
filteredDocs.length
@ -54,7 +53,6 @@ export async function scrapeHelper(
returnCode: 402,
};
}
}
return {
success: true,

19
apps/api/src/lib/withAuth.ts Normal file
View File

@ -0,0 +1,19 @@
import { AuthResponse } from "../../src/types";
export function withAuth<T extends AuthResponse, U extends any[]>(
originalFunction: (...args: U) => Promise<T>
) {
return async function (...args: U): Promise<T> {
if (process.env.USE_DB_AUTHENTICATION === "false") {
console.warn("WARNING - You're bypassing authentication");
return { success: true } as T;
} else {
try {
return await originalFunction(...args);
} catch (error) {
console.error("Error in withAuth function: ", error);
return { success: false, error: error.message } as T;
}
}
};
}

10
apps/api/src/services/billing/credit_billing.ts
View File

@ -1,7 +1,12 @@
import { withAuth } from "../../lib/withAuth";
import { supabase_service } from "../supabase";
const FREE_CREDITS = 100;
export async function billTeam(team_id: string, credits: number) {
return withAuth(supaBillTeam)(team_id, credits);
}
export async function supaBillTeam(team_id: string, credits: number) {
if (team_id === "preview") {
return { success: true, message: "Preview team, no credits used" };
}
@ -52,8 +57,11 @@ export async function billTeam(team_id: string, credits: number) {
return { success: true, credit_usage };
}
// if team has enough credits for the operation, return true, else return false
export async function checkTeamCredits(team_id: string, credits: number) {
return withAuth(supaCheckTeamCredits)(team_id, credits);
}
// if team has enough credits for the operation, return true, else return false
export async function supaCheckTeamCredits(team_id: string, credits: number) {
if (team_id === "preview") {
return { success: true, message: "Preview team, no credits used" };
}

51
apps/api/src/services/supabase.ts
View File

@ -1,4 +1,4 @@
import { createClient, SupabaseClient } from '@supabase/supabase-js';
import { createClient, SupabaseClient } from "@supabase/supabase-js";
// SupabaseService class initializes the Supabase client conditionally based on environment variables.
class SupabaseService {
@ -7,15 +7,17 @@ class SupabaseService {
constructor() {
const supabaseUrl = process.env.SUPABASE_URL;
const supabaseServiceToken = process.env.SUPABASE_SERVICE_TOKEN;
// Only initialize the Supabase client if both URL and Service Token are provided.
if (process.env.USE_DB_AUTHENTICATION === "false") {
// Warn the user that Authentication is disabled by setting the client to null
console.warn("\x1b[33mAuthentication is disabled. Supabase client will not be initialized.\x1b[0m");
console.warn(
"\x1b[33mAuthentication is disabled. Supabase client will not be initialized.\x1b[0m"
);
this.client = null;
} else if (!supabaseUrl || !supabaseServiceToken) {
console.error("\x1b[31mSupabase environment variables aren't configured correctly. Supabase client will not be initialized. Fix ENV configuration or disable DB authentication with USE_DB_AUTHENTICATION env variable\x1b[0m");
console.error(
"\x1b[31mSupabase environment variables aren't configured correctly. Supabase client will not be initialized. Fix ENV configuration or disable DB authentication with USE_DB_AUTHENTICATION env variable\x1b[0m"
);
} else {
this.client = createClient(supabaseUrl, supabaseServiceToken);
}
@ -29,21 +31,26 @@ class SupabaseService {
// Using a Proxy to handle dynamic access to the Supabase client or service methods.
// This approach ensures that if Supabase is not configured, any attempt to use it will result in a clear error.
export const supabase_service: SupabaseClient = new Proxy(new SupabaseService(), {
get: function (target, prop, receiver) {
const client = target.getClient();
// If the Supabase client is not initialized, intercept property access to provide meaningful error feedback.
if (client === null) {
console.error("Attempted to access Supabase client when it's not configured.");
return () => {
throw new Error("Supabase client is not configured.");
};
}
// Direct access to SupabaseService properties takes precedence.
if (prop in target) {
return Reflect.get(target, prop, receiver);
}
// Otherwise, delegate access to the Supabase client.
return Reflect.get(client, prop, receiver);
export const supabase_service: SupabaseClient = new Proxy(
new SupabaseService(),
{
get: function (target, prop, receiver) {
const client = target.getClient();
// If the Supabase client is not initialized, intercept property access to provide meaningful error feedback.
if (client === null) {
console.error(
"Attempted to access Supabase client when it's not configured."
);
return () => {
throw new Error("Supabase client is not configured.");
};
}
// Direct access to SupabaseService properties takes precedence.
if (prop in target) {
return Reflect.get(target, prop, receiver);
}
// Otherwise, delegate access to the Supabase client.
return Reflect.get(client, prop, receiver);
},
}
}) as unknown as SupabaseClient;
) as unknown as SupabaseClient;

10
apps/api/src/types.ts
View File

@ -25,7 +25,6 @@ export interface WebScraperOptions {
origin?: string;
}
export interface FirecrawlJob {
success: boolean;
message: string;
@ -40,8 +39,6 @@ export interface FirecrawlJob {
origin: string;
}
export enum RateLimiterMode {
Crawl = "crawl",
CrawlStatus = "crawl-status",
@ -49,4 +46,9 @@ export enum RateLimiterMode {
Preview = "preview",
}
export interface AuthResponse {
success: boolean;
team_id?: string;
error?: string;
status?: number;
}