diff --git a/apps/js-sdk/example.js b/apps/js-sdk/example.ts similarity index 80% rename from apps/js-sdk/example.js rename to apps/js-sdk/example.ts index 7077b4c6..11a58510 100644 --- a/apps/js-sdk/example.js +++ b/apps/js-sdk/example.ts @@ -17,4 +17,7 @@ while (true) { await new Promise(resolve => setTimeout(resolve, 1000)); // wait 1 second } -console.log(job.data[0].content); \ No newline at end of file +console.log(job.data); + + +const a =await app.scrapeUrl('mendable.ai', {crawlerOptions: {excludes: ['blog/*'], limit: 5}}); \ No newline at end of file diff --git a/apps/js-sdk/firecrawl/build/index.js b/apps/js-sdk/firecrawl/build/index.js index 9d8237b4..e1b839ee 100644 --- a/apps/js-sdk/firecrawl/build/index.js +++ b/apps/js-sdk/firecrawl/build/index.js @@ -8,8 +8,6 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge }); }; import axios from 'axios'; -import dotenv from 'dotenv'; -dotenv.config(); /** * Main class for interacting with the Firecrawl API. */ @@ -19,7 +17,7 @@ export default class FirecrawlApp { * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance. */ constructor({ apiKey = null }) { - this.apiKey = apiKey || process.env.FIRECRAWL_API_KEY || ''; + this.apiKey = apiKey || ''; if (!this.apiKey) { throw new Error('No API key provided'); } diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json index a8275f7b..8c6c4365 100644 --- a/apps/js-sdk/firecrawl/package.json +++ b/apps/js-sdk/firecrawl/package.json @@ -1,6 +1,6 @@ { "name": "@mendable/firecrawl-js", - "version": "0.0.16", + "version": "0.0.17-beta.1", "description": "JavaScript SDK for Firecrawl API", "main": "build/index.js", "types": "types/index.d.ts", @@ -8,6 +8,7 @@ "scripts": { "build": "tsc", "publish": "npm run build && npm publish --access public", + "publish:beta": "npm run build && npm publish --access public --tag beta", "test": "jest src/**/*.test.ts" }, "repository": { diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.ts index aea15f83..4fcb0091 100644 --- a/apps/js-sdk/firecrawl/src/index.ts +++ b/apps/js-sdk/firecrawl/src/index.ts @@ -19,7 +19,7 @@ export interface Params { */ export interface ScrapeResponse { success: boolean; - data?: any; + data?: Document; error?: string; } @@ -28,7 +28,7 @@ export interface ScrapeResponse { */ export interface SearchResponse { success: boolean; - data?: any; + data?: Document[] | { url: string, title: string, description: string }[]; error?: string; } /** @@ -37,7 +37,7 @@ export interface SearchResponse { export interface CrawlResponse { success: boolean; jobId?: string; - data?: any; + data?: Document[]; error?: string; } @@ -52,6 +52,28 @@ export interface JobStatusResponse { error?: string; } +/** + * Return type for scraping, crawling and search operations. + */ +export interface Document{ + id?: string; + content: string; + markdown?: string; + html?: string; + llm_extraction?: Record; + createdAt?: Date; + updatedAt?: Date; + type?: string; + metadata: { + sourceURL?: string; + [key: string]: any; + }; + childrenLinks?: string[]; + provider?: string; + url?: string; // Used only in /search for now +} + + /** * Main class for interacting with the Firecrawl API. */ diff --git a/apps/js-sdk/firecrawl/types/index.d.ts b/apps/js-sdk/firecrawl/types/index.d.ts index 7f79d644..c7b66e5e 100644 --- a/apps/js-sdk/firecrawl/types/index.d.ts +++ b/apps/js-sdk/firecrawl/types/index.d.ts @@ -16,7 +16,7 @@ export interface Params { */ export interface ScrapeResponse { success: boolean; - data?: any; + data?: Document; error?: string; } /** @@ -24,7 +24,11 @@ export interface ScrapeResponse { */ export interface SearchResponse { success: boolean; - data?: any; + data?: Document[] | { + url: string; + title: string; + description: string; + }[]; error?: string; } /** @@ -33,7 +37,7 @@ export interface SearchResponse { export interface CrawlResponse { success: boolean; jobId?: string; - data?: any; + data?: Document[]; error?: string; } /** @@ -46,6 +50,26 @@ export interface JobStatusResponse { data?: any; error?: string; } +/** + * Return type for scraping, crawling and search operations. + */ +export interface Document { + id?: string; + content: string; + markdown?: string; + html?: string; + llm_extraction?: Record; + createdAt?: Date; + updatedAt?: Date; + type?: string; + metadata: { + sourceURL?: string; + [key: string]: any; + }; + childrenLinks?: string[]; + provider?: string; + url?: string; +} /** * Main class for interacting with the Firecrawl API. */ diff --git a/apps/js-sdk/package-lock.json b/apps/js-sdk/package-lock.json index 363f3013..042fdf35 100644 --- a/apps/js-sdk/package-lock.json +++ b/apps/js-sdk/package-lock.json @@ -9,17 +9,16 @@ "version": "1.0.0", "license": "ISC", "dependencies": { - "@mendable/firecrawl-js": "^0.0.15", + "@mendable/firecrawl-js": "^0.0.17-beta.1", "axios": "^1.6.8" } }, "node_modules/@mendable/firecrawl-js": { - "version": "0.0.15", - "resolved": "https://registry.npmjs.org/@mendable/firecrawl-js/-/firecrawl-js-0.0.15.tgz", - "integrity": "sha512-e3iCCrLIiEh+jEDerGV9Uhdkn8ymo+sG+k3osCwPg51xW1xUdAnmlcHrcJoR43RvKXdvD/lqoxg8odUEsqyH+w==", + "version": "0.0.17-beta.1", + "resolved": "https://registry.npmjs.org/@mendable/firecrawl-js/-/firecrawl-js-0.0.17-beta.1.tgz", + "integrity": "sha512-GxBiYIJoFn4ddKTiN6WaFfCapRp3qNDNTWw90sV3kY+YGs5s40oIJ7bkDcI1Am59QhntiKzKB4IcjpV1LVfQTA==", "dependencies": { - "axios": "^1.6.8", - "dotenv": "^16.4.5" + "axios": "^1.6.8" } }, "node_modules/asynckit": { @@ -56,17 +55,6 @@ "node": ">=0.4.0" } }, - "node_modules/dotenv": { - "version": "16.4.5", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", - "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, "node_modules/follow-redirects": { "version": "1.15.6", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", diff --git a/apps/js-sdk/package.json b/apps/js-sdk/package.json index 563e1e39..fea4f3c5 100644 --- a/apps/js-sdk/package.json +++ b/apps/js-sdk/package.json @@ -11,7 +11,7 @@ "author": "", "license": "ISC", "dependencies": { - "@mendable/firecrawl-js": "^0.0.15", + "@mendable/firecrawl-js": "^0.0.17-beta.1", "axios": "^1.6.8" } }