Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Nick: totalCount -> total, completedCount -> completed

Browse Source
This commit is contained in:
Nicolas 2024-08-28 14:46:53 -03:00
parent 2a56f89f50
commit 4edd9a34c7
10 changed files with 31 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -76,7 +76,7 @@ curl -X GET https://api.firecrawl.dev/v1/crawl/123-456-789 \
```json
{
"status": "completed",
"totalCount": 36,
"total": 36,
"creditsUsed": 36,
"expiresAt": "2024-00-00T00:00:00.000Z",
"data": [

3
apps/api/src/controllers/v1/crawl-status-ws.ts
View File

@ -102,7 +102,8 @@ async function crawlStatusWS(ws: WebSocket, req: RequestWithAuth<CrawlStatusPara
type: "catchup",
data: {
status,
totalCount: jobIDs.length,
total: jobIDs.length,
completed: doneJobIDs.length,
creditsUsed: jobIDs.length,
expiresAt: (await getCrawlExpiry(req.params.jobId)).toISOString(),
data: data.map(x => legacyDocumentConverter(x)),

3
apps/api/src/controllers/v1/crawl-status.ts
View File

@ -102,7 +102,8 @@ export async function crawlStatusController(req: RequestWithAuth<CrawlStatusPara
res.status(200).json({
status,
totalCount: jobIDs.length,
completed: doneJobsLength,
total: jobIDs.length,
creditsUsed: jobIDs.length,
expiresAt: (await getCrawlExpiry(req.params.jobId)).toISOString(),
next:

3
apps/api/src/controllers/v1/types.ts
View File

@ -221,7 +221,8 @@ export type CrawlStatusResponse =
| ErrorResponse
| {
status: "scraping" | "completed" | "failed" | "cancelled";
totalCount: number;
completed: number;
total: number;
creditsUsed: number;
expiresAt: string;
next?: string;

2
apps/js-sdk/firecrawl/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
"version": "0.0.36",
"version": "1.0.0",
"description": "JavaScript SDK for Firecrawl API",
"main": "build/cjs/index.js",
"types": "types/index.d.ts",

16
apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts
View File

@ -129,8 +129,8 @@ describe('FirecrawlApp E2E Tests', () => {
const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
const response = await app.crawlUrl('https://roastmywebsite.ai', {}, true, 30) as CrawlStatusResponse;
expect(response).not.toBeNull();
expect(response).toHaveProperty("totalCount");
expect(response.totalCount).toBeGreaterThan(0);
expect(response).toHaveProperty("total");
expect(response.total).toBeGreaterThan(0);
expect(response).toHaveProperty("creditsUsed");
expect(response.creditsUsed).toBeGreaterThan(0);
expect(response).toHaveProperty("expiresAt");
@ -175,8 +175,8 @@ describe('FirecrawlApp E2E Tests', () => {
}
} as CrawlParams, true, 30) as CrawlStatusResponse;
expect(response).not.toBeNull();
expect(response).toHaveProperty("totalCount");
expect(response.totalCount).toBeGreaterThan(0);
expect(response).toHaveProperty("total");
expect(response.total).toBeGreaterThan(0);
expect(response).toHaveProperty("creditsUsed");
expect(response.creditsUsed).toBeGreaterThan(0);
expect(response).toHaveProperty("expiresAt");
@ -231,12 +231,12 @@ describe('FirecrawlApp E2E Tests', () => {
expect(statusResponse).not.toHaveProperty("partial_data"); // v0
expect(statusResponse).not.toHaveProperty("current"); // v0
expect(statusResponse).toHaveProperty("data");
expect(statusResponse).toHaveProperty("totalCount");
expect(statusResponse).toHaveProperty("total");
expect(statusResponse).toHaveProperty("creditsUsed");
expect(statusResponse).toHaveProperty("expiresAt");
expect(statusResponse).toHaveProperty("status");
expect(statusResponse).toHaveProperty("next");
expect(statusResponse.totalCount).toBeGreaterThan(0);
expect(statusResponse.total).toBeGreaterThan(0);
expect(statusResponse.creditsUsed).toBeGreaterThan(0);
expect(statusResponse.expiresAt.getTime()).toBeGreaterThan(Date.now());
expect(statusResponse.status).toBe("scraping");
@ -246,8 +246,8 @@ describe('FirecrawlApp E2E Tests', () => {
}
expect(statusResponse).not.toBeNull();
expect(statusResponse).toHaveProperty("totalCount");
expect(statusResponse.totalCount).toBeGreaterThan(0);
expect(statusResponse).toHaveProperty("total");
expect(statusResponse.total).toBeGreaterThan(0);
expect(statusResponse).toHaveProperty("creditsUsed");
expect(statusResponse.creditsUsed).toBeGreaterThan(0);
expect(statusResponse).toHaveProperty("expiresAt");

6
apps/js-sdk/firecrawl/src/index.ts
View File

@ -235,7 +235,8 @@ export interface CrawlResponseV0 {
*/
export interface CrawlStatusResponse {
success: boolean;
totalCount: number;
total: number;
completed: number;
creditsUsed: number;
expiresAt: Date;
status: "scraping" | "completed" | "failed";
@ -530,7 +531,8 @@ export default class FirecrawlApp<T extends "v0" | "v1"> {
return ({
success: true,
status: response.data.status,
totalCount: response.data.totalCount,
total: response.data.total,
completed: response.data.completed,
creditsUsed: response.data.creditsUsed,
expiresAt: new Date(response.data.expiresAt),
next: response.data.next,

2
apps/python-sdk/firecrawl/__init__.py
View File

@ -13,7 +13,7 @@ import os
from .firecrawl import FirecrawlApp
__version__ = "0.0.16"
__version__ = "1.0.0"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")

16
apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py
View File

@ -147,8 +147,8 @@ def test_crawl_url_wait_for_completion_e2e():
app = FirecrawlApp(api_url=API_URL, api_key=TEST_API_KEY)
response = app.crawl_url('https://roastmywebsite.ai', {'excludePaths': ['blog/*']}, True, 30)
assert response is not None
assert 'totalCount' in response
assert response['totalCount'] > 0
assert 'total' in response
assert response['total'] > 0
assert 'creditsUsed' in response
assert response['creditsUsed'] > 0
assert 'expiresAt' in response
@ -192,8 +192,8 @@ def test_crawl_url_with_options_and_wait_for_completion():
}
}, True, 30)
assert response is not None
assert 'totalCount' in response
assert response['totalCount'] > 0
assert 'total' in response
assert response['total'] > 0
assert 'creditsUsed' in response
assert response['creditsUsed'] > 0
assert 'expiresAt' in response
@ -247,12 +247,12 @@ def test_check_crawl_status_e2e():
assert 'partial_data' not in status_response
assert 'current' not in status_response
assert 'data' in status_response
assert 'totalCount' in status_response
assert 'total' in status_response
assert 'creditsUsed' in status_response
assert 'expiresAt' in status_response
assert 'status' in status_response
assert 'next' in status_response
assert status_response['totalCount'] > 0
assert status_response['total'] > 0
assert status_response['creditsUsed'] > 0
assert datetime.strptime(status_response['expiresAt'], '%Y-%m-%dT%H:%M:%S.%fZ') > datetime.now()
assert status_response['status'] == 'scraping'
@ -261,8 +261,8 @@ def test_check_crawl_status_e2e():
checks += 1
assert status_response is not None
assert 'totalCount' in status_response
assert status_response['totalCount'] > 0
assert 'total' in status_response
assert status_response['total'] > 0
assert 'creditsUsed' in status_response
assert status_response['creditsUsed'] > 0
assert 'expiresAt' in status_response

3
apps/python-sdk/firecrawl/firecrawl.py
View File

@ -211,7 +211,8 @@ class FirecrawlApp:
return {
'success': True,
'status': data.get('status'),
'totalCount': data.get('totalCount'),
'total': data.get('total'),
'completed': data.get('completed'),
'creditsUsed': data.get('creditsUsed'),
'expiresAt': data.get('expiresAt'),
'next': data.get('next'),