Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Added e2e tests

Browse Source
This commit is contained in:
rafaelsideguide 2024-05-17 15:37:47 -03:00
parent d4574851be
commit 54049be539
3 changed files with 66 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/api/.env.example
View File

@ -16,6 +16,8 @@ SUPABASE_SERVICE_TOKEN=
# Other Optionals
TEST_API_KEY= # use if you've set up authentication and want to test with a real API key
RATE_LIMIT_TEST_API_KEY_SCRAPE= # set if you'd like to test the scraping rate limit
RATE_LIMIT_TEST_API_KEY_CRAWL= # set if you'd like to test the crawling rate limit
SCRAPING_BEE_API_KEY= #Set if you'd like to use scraping Be to handle JS blocking
OPENAI_API_KEY= # add for LLM dependednt features (image alt generation, etc.)
BULL_AUTH_KEY= #

61
apps/api/src/__tests__/e2e_withAuth/index.test.ts
View File

@ -518,4 +518,65 @@ describe("E2E Tests for API Routes", () => {
expect(response.body).toHaveProperty("isProduction");
});
});
describe("Rate Limiter", () => {
it("should return 429 when rate limit is exceeded for preview token", async () => {
for (let i = 0; i < 5; i++) {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer this_is_just_a_preview_token`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(200);
}
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer this_is_just_a_preview_token`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(429);
}, 60000);
});
it("should return 429 when rate limit is exceeded for API key", async () => {
for (let i = 0; i < parseInt(process.env.RATE_LIMIT_TEST_API_KEY_SCRAPE); i++) {
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(200);
}
const response = await request(TEST_URL)
.post("/v0/scrape")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(429);
}, 60000);
it("should return 429 when rate limit is exceeded for API key", async () => {
for (let i = 0; i < parseInt(process.env.RATE_LIMIT_TEST_API_KEY_CRAWL); i++) {
const response = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(200);
}
const response = await request(TEST_URL)
.post("/v0/crawl")
.set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
.set("Content-Type", "application/json")
.send({ url: "https://firecrawl.dev" });
expect(response.statusCode).toBe(429);
}, 60000);
});

4
apps/api/src/controllers/auth.ts
View File

@ -41,7 +41,7 @@ export async function supaAuthenticateUser(
let normalizedApi: string;
if (token == "this_is_just_a_preview_token") {
rateLimiter = await getRateLimiter(RateLimiterMode.Preview, token);
rateLimiter = getRateLimiter(RateLimiterMode.Preview, token);
} else {
normalizedApi = parseApi(token);
@ -103,7 +103,7 @@ export async function supaAuthenticateUser(
}
try {
rateLimiter.consume(iptoken);
await rateLimiter.consume(iptoken);
} catch (rateLimiterRes) {
console.error(rateLimiterRes);
return {