Austin/firecrawl
1
0
Fork 0
mirror of https://github.com/mendableai/firecrawl.git synced 2024-11-16 11:42:24 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #391 from jhoseph88/feat/issue-387

Browse Source 
[Feat] Pass along current, total, current_step, and current_url in js sdk
This commit is contained in:
Rafael Miller 2024-07-16 15:56:42 -03:00 committed by GitHub
commit db0545014f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 33 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
apps/js-sdk/firecrawl/build/index.js
View File

@ -157,8 +157,14 @@ export default class FirecrawlApp {
return {
success: true,
status: response.data.status,
current: response.data.current,
current_url: response.data.current_url,
current_step: response.data.current_step,
total: response.data.total,
data: response.data.data,
partial_data: !response.data.data ? response.data.partial_data : undefined,
partial_data: !response.data.data
? response.data.partial_data
: undefined,
};
}
else {
@ -171,6 +177,10 @@ export default class FirecrawlApp {
return {
success: false,
status: "unknown",
current: 0,
current_url: "",
current_step: "",
total: 0,
error: "Internal server error.",
};
});
@ -180,7 +190,7 @@ export default class FirecrawlApp {
* @returns {AxiosRequestHeaders} The prepared headers.
*/
prepareHeaders(idempotencyKey) {
return Object.assign({ 'Content-Type': 'application/json', 'Authorization': `Bearer ${this.apiKey}` }, (idempotencyKey ? { 'x-idempotency-key': idempotencyKey } : {}));
return Object.assign({ "Content-Type": "application/json", Authorization: `Bearer ${this.apiKey}` }, (idempotencyKey ? { "x-idempotency-key": idempotencyKey } : {}));
}
/**
* Sends a POST request to the specified URL.

4
apps/js-sdk/firecrawl/package-lock.json generated
View File

@ -1,12 +1,12 @@
{
"name": "@mendable/firecrawl-js",
"version": "0.0.28",
"version": "0.0.29",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@mendable/firecrawl-js",
"version": "0.0.28",
"version": "0.0.29",
"license": "MIT",
"dependencies": {
"axios": "^1.6.8",

2
apps/js-sdk/firecrawl/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
"version": "0.0.28",
"version": "0.0.29",
"description": "JavaScript SDK for Firecrawl API",
"main": "build/index.js",
"types": "types/index.d.ts",

8
apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts
View File

@ -1,7 +1,7 @@
import FirecrawlApp from '../../index';
import { v4 as uuidv4 } from 'uuid';
import dotenv from 'dotenv';
import { describe, test, expect } from '@jest/globals';
dotenv.config();
@ -9,7 +9,7 @@ const TEST_API_KEY = process.env.TEST_API_KEY;
const API_URL = "http://127.0.0.1:3002";
describe('FirecrawlApp E2E Tests', () => {
test.concurrent('should throw error for no API key', () => {
test.concurrent('should throw error for no API key', async () => {
expect(() => {
new FirecrawlApp({ apiKey: null, apiUrl: API_URL });
}).toThrow("No API key provided");
@ -107,12 +107,16 @@ describe('FirecrawlApp E2E Tests', () => {
while (statusResponse.status === 'active' && checks < maxChecks) {
await new Promise(resolve => setTimeout(resolve, 1000));
expect(statusResponse.partial_data).not.toBeNull();
expect(statusResponse.current).toBeGreaterThanOrEqual(1);
statusResponse = await app.checkCrawlStatus(response.jobId);
checks++;
}
expect(statusResponse).not.toBeNull();
expect(statusResponse.success).toBe(true);
expect(statusResponse.status).toBe('completed');
expect(statusResponse.total).toEqual(statusResponse.current);
expect(statusResponse.current_step).not.toBeNull();
expect(statusResponse?.data?.length).toBeGreaterThan(0);
}, 35000); // 35 seconds timeout

12
apps/js-sdk/firecrawl/src/index.ts
View File

@ -100,6 +100,10 @@ export interface CrawlResponse {
export interface JobStatusResponse {
success: boolean;
status: string;
current?: number;
current_url?: string;
current_step?: string;
total?: number;
jobId?: string;
data?: FirecrawlDocument[];
partial_data?: FirecrawlDocument[];
@ -287,6 +291,10 @@ export default class FirecrawlApp {
return {
success: true,
status: response.data.status,
current: response.data.current,
current_url: response.data.current_url,
current_step: response.data.current_step,
total: response.data.total,
data: response.data.data,
partial_data: !response.data.data
? response.data.partial_data
@ -301,6 +309,10 @@ export default class FirecrawlApp {
return {
success: false,
status: "unknown",
current: 0,
current_url: "",
current_step: "",
total: 0,
error: "Internal server error.",
};
}